From d96377e77411c40339d26d832894b06cc945c351 Mon Sep 17 00:00:00 2001
From: Nam Thanh Nguyen <35841468+jkelol111@users.noreply.github.com>
Date: Tue, 16 Feb 2021 15:17:49 +0800
Subject: [PATCH] :rocket: Too many changes!
- Added FileExcludeSearch file property: exclude the file from latest, random and search.
- Added replicator script: easy archives and restores for Iamages server.
- Added database upgrader: upgrade current database to latest format easily. (From format 1 -> 2).
- Slight update to ToS to comply with UberSpace house rules.
- Changed name of database maker script.
- Server and scripts now check for storage format version.
- Updated dependencies.
---
.gitignore | 2 +
Pipfile.lock | 180 +++++++++++------------
iamagesdb_create.py => iamages_mkdb.py | 8 +-
iamages_replicator.py | 189 +++++++++++++++++++++++++
iamages_server.py | 37 ++++-
iamages_updb.py | 54 +++++++
iamagesdb.sql | 5 +-
requirements.txt | 8 +-
templates/tos.html | 5 +-
9 files changed, 386 insertions(+), 102 deletions(-)
rename iamagesdb_create.py => iamages_mkdb.py (85%)
create mode 100644 iamages_replicator.py
create mode 100644 iamages_updb.py
diff --git a/.gitignore b/.gitignore
index 4d6cadb..a46361c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,7 @@
iamages_storage
+iamages_replicated
__pycache__
.vscode
.DS_Store
servercfg.json
+replicatorcfg.json
diff --git a/Pipfile.lock b/Pipfile.lock
index 0ea7cb0..ca8d749 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -46,44 +46,45 @@
},
"cffi": {
"hashes": [
- "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e",
- "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d",
- "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a",
- "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec",
- "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362",
- "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668",
- "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c",
- "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b",
- "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06",
- "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698",
- "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2",
- "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c",
- "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7",
- "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009",
- "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03",
- "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b",
- "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909",
- "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53",
- "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35",
- "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26",
- "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b",
- "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01",
- "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb",
- "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293",
- "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd",
- "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d",
- "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3",
- "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d",
- "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e",
- "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca",
- "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d",
- "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775",
- "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375",
- "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b",
- "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b",
- "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"
+ "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813",
+ "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06",
+ "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea",
+ "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee",
+ "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396",
+ "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73",
+ "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315",
+ "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1",
+ "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49",
+ "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892",
+ "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482",
+ "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058",
+ "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5",
+ "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53",
+ "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045",
+ "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3",
+ "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5",
+ "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e",
+ "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c",
+ "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369",
+ "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827",
+ "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053",
+ "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa",
+ "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4",
+ "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322",
+ "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132",
+ "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62",
+ "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa",
+ "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0",
+ "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396",
+ "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e",
+ "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991",
+ "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6",
+ "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1",
+ "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406",
+ "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d",
+ "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"
],
- "version": "==1.14.4"
+ "version": "==1.14.5"
},
"click": {
"hashes": [
@@ -300,55 +301,55 @@
},
"sqlalchemy": {
"hashes": [
- "sha256:04f995fcbf54e46cddeb4f75ce9dfc17075d6ae04ac23b2bacb44b3bc6f6bf11",
- "sha256:0c6406a78a714a540d980a680b86654feadb81c8d0eecb59f3d6c554a4c69f19",
- "sha256:0c72b90988be749e04eff0342dcc98c18a14461eb4b2ad59d611b57b31120f90",
- "sha256:108580808803c7732f34798eb4a329d45b04c562ed83ee90f09f6a184a42b766",
- "sha256:1418f5e71d6081aa1095a1d6b567a562d2761996710bdce9b6e6ba20a03d0864",
- "sha256:17610d573e698bf395afbbff946544fbce7c5f4ee77b5bcb1f821b36345fae7a",
- "sha256:216ba5b4299c95ed179b58f298bda885a476b16288ab7243e89f29f6aeced7e0",
- "sha256:2ff132a379838b1abf83c065be54cef32b47c987aedd06b82fc76476c85225eb",
- "sha256:314f5042c0b047438e19401d5f29757a511cfc2f0c40d28047ca0e4c95eabb5b",
- "sha256:318b5b727e00662e5fc4b4cd2bf58a5116d7c1b4dd56ffaa7d68f43458a8d1ed",
- "sha256:3ab5b44a07b8c562c6dcb7433c6a6c6e03266d19d64f87b3333eda34e3b9936b",
- "sha256:426ece890153ccc52cc5151a1a0ed540a5a7825414139bb4c95a868d8da54a52",
- "sha256:491fe48adc07d13e020a8b07ef82eefc227003a046809c121bea81d3dbf1832d",
- "sha256:4a84c7c7658dd22a33dab2e2aa2d17c18cb004a42388246f2e87cb4085ef2811",
- "sha256:54da615e5b92c339e339fe8536cce99fe823b6ed505d4ea344852aefa1c205fb",
- "sha256:5a7f224cdb7233182cec2a45d4c633951268d6a9bcedac37abbf79dd07012aea",
- "sha256:61628715931f4962e0cdb2a7c87ff39eea320d2aa96bd471a3c293d146f90394",
- "sha256:62285607a5264d1f91590abd874d6a498e229d5840669bd7d9f654cfaa599bd0",
- "sha256:62fb881ba51dbacba9af9b779211cf9acff3442d4f2993142015b22b3cd1f92a",
- "sha256:68428818cf80c60dc04aa0f38da20ad39b28aba4d4d199f949e7d6e04444ea86",
- "sha256:6aaa13ee40c4552d5f3a59f543f0db6e31712cc4009ec7385407be4627259d41",
- "sha256:70121f0ae48b25ef3e56e477b88cd0b0af0e1f3a53b5554071aa6a93ef378a03",
- "sha256:715b34578cc740b743361f7c3e5f584b04b0f1344f45afc4e87fbac4802eb0a0",
- "sha256:758fc8c4d6c0336e617f9f6919f9daea3ab6bb9b07005eda9a1a682e24a6cacc",
- "sha256:7d4b8de6bb0bc736161cb0bbd95366b11b3eb24dd6b814a143d8375e75af9990",
- "sha256:81d8d099a49f83111cce55ec03cc87eef45eec0d90f9842b4fc674f860b857b0",
- "sha256:888d5b4b5aeed0d3449de93ea80173653e939e916cc95fe8527079e50235c1d2",
- "sha256:95bde07d19c146d608bccb9b16e144ec8f139bcfe7fd72331858698a71c9b4f5",
- "sha256:9bf572e4f5aa23f88dd902f10bb103cb5979022a38eec684bfa6d61851173fec",
- "sha256:bab5a1e15b9466a25c96cda19139f3beb3e669794373b9ce28c4cf158c6e841d",
- "sha256:bd4b1af45fd322dcd1fb2a9195b4f93f570d1a5902a842e3e6051385fac88f9c",
- "sha256:bde677047305fe76c7ee3e4492b545e0018918e44141cc154fe39e124e433991",
- "sha256:c389d7cc2b821853fb018c85457da3e7941db64f4387720a329bc7ff06a27963",
- "sha256:d055ff750fcab69ca4e57b656d9c6ad33682e9b8d564f2fbe667ab95c63591b0",
- "sha256:d53f59744b01f1440a1b0973ed2c3a7de204135c593299ee997828aad5191693",
- "sha256:f115150cc4361dd46153302a640c7fa1804ac207f9cc356228248e351a8b4676",
- "sha256:f1e88b30da8163215eab643962ae9d9252e47b4ea53404f2c4f10f24e70ddc62",
- "sha256:f8191fef303025879e6c3548ecd8a95aafc0728c764ab72ec51a0bdf0c91a341"
+ "sha256:040bdfc1d76a9074717a3f43455685f781c581f94472b010cd6c4754754e1862",
+ "sha256:1fe5d8d39118c2b018c215c37b73fd6893c3e1d4895be745ca8ff6eb83333ed3",
+ "sha256:23927c3981d1ec6b4ea71eb99d28424b874d9c696a21e5fbd9fa322718be3708",
+ "sha256:24f9569e82a009a09ce2d263559acb3466eba2617203170e4a0af91e75b4f075",
+ "sha256:2578dbdbe4dbb0e5126fb37ffcd9793a25dcad769a95f171a2161030bea850ff",
+ "sha256:269990b3ab53cb035d662dcde51df0943c1417bdab707dc4a7e4114a710504b4",
+ "sha256:29cccc9606750fe10c5d0e8bd847f17a97f3850b8682aef1f56f5d5e1a5a64b1",
+ "sha256:37b83bf81b4b85dda273aaaed5f35ea20ad80606f672d94d2218afc565fb0173",
+ "sha256:63677d0c08524af4c5893c18dbe42141de7178001360b3de0b86217502ed3601",
+ "sha256:639940bbe1108ac667dcffc79925db2966826c270112e9159439ab6bb14f8d80",
+ "sha256:6a939a868fdaa4b504e8b9d4a61f21aac11e3fecc8a8214455e144939e3d2aea",
+ "sha256:6b8b8c80c7f384f06825612dd078e4a31f0185e8f1f6b8c19e188ff246334205",
+ "sha256:6c9e6cc9237de5660bcddea63f332428bb83c8e2015c26777281f7ffbd2efb84",
+ "sha256:6ec1044908414013ebfe363450c22f14698803ce97fbb47e53284d55c5165848",
+ "sha256:6fca33672578666f657c131552c4ef8979c1606e494f78cd5199742dfb26918b",
+ "sha256:751934967f5336a3e26fc5993ccad1e4fee982029f9317eb6153bc0bc3d2d2da",
+ "sha256:8be835aac18ec85351385e17b8665bd4d63083a7160a017bef3d640e8e65cadb",
+ "sha256:927ce09e49bff3104459e1451ce82983b0a3062437a07d883a4c66f0b344c9b5",
+ "sha256:94208867f34e60f54a33a37f1c117251be91a47e3bfdb9ab8a7847f20886ad06",
+ "sha256:94f667d86be82dd4cb17d08de0c3622e77ca865320e0b95eae6153faa7b4ecaf",
+ "sha256:9e9c25522933e569e8b53ccc644dc993cab87e922fb7e142894653880fdd419d",
+ "sha256:a0e306e9bb76fd93b29ae3a5155298e4c1b504c7cbc620c09c20858d32d16234",
+ "sha256:a8bfc1e1afe523e94974132d7230b82ca7fa2511aedde1f537ec54db0399541a",
+ "sha256:ac2244e64485c3778f012951fdc869969a736cd61375fde6096d08850d8be729",
+ "sha256:b4b0e44d586cd64b65b507fa116a3814a1a53d55dce4836d7c1a6eb2823ff8d1",
+ "sha256:baeb451ee23e264de3f577fee5283c73d9bbaa8cb921d0305c0bbf700094b65b",
+ "sha256:c7dc052432cd5d060d7437e217dd33c97025287f99a69a50e2dc1478dd610d64",
+ "sha256:d1a85dfc5dee741bf49cb9b6b6b8d2725a268e4992507cf151cba26b17d97c37",
+ "sha256:d90010304abb4102123d10cbad2cdf2c25a9f2e66a50974199b24b468509bad5",
+ "sha256:ddfb511e76d016c3a160910642d57f4587dc542ce5ee823b0d415134790eeeb9",
+ "sha256:e273367f4076bd7b9a8dc2e771978ef2bfd6b82526e80775a7db52bff8ca01dd",
+ "sha256:e5bb3463df697279e5459a7316ad5a60b04b0107f9392e88674d0ece70e9cf70",
+ "sha256:e8a1750b44ad6422ace82bf3466638f1aa0862dbb9689690d5f2f48cce3476c8",
+ "sha256:eab063a70cca4a587c28824e18be41d8ecc4457f8f15b2933584c6c6cccd30f0",
+ "sha256:ecce8c021894a77d89808222b1ff9687ad84db54d18e4bd0500ca766737faaf6",
+ "sha256:f4d972139d5000105fcda9539a76452039434013570d6059993120dc2a65e447",
+ "sha256:fd3b96f8c705af8e938eaa99cbd8fd1450f632d38cad55e7367c33b263bf98ec",
+ "sha256:fdd2ed7395df8ac2dbb10cefc44737b66c6a5cd7755c92524733d7a443e5b7e2"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.3.22"
+ "version": "==1.3.23"
},
"starlette": {
"hashes": [
- "sha256:5268ef5d4904ec69582d5fd207b869a5aa0cd59529848ba4cf429b06e3ced99a",
- "sha256:d2f55fb835378442b812637ed3e3fcef3d3e22d292fcb8400fa48d2473202411"
+ "sha256:3c8e48e52736b3161e34c9f0e8153b4f32ec5d8995a3ee1d59410d92f75162ed",
+ "sha256:7d49f4a27f8742262ef1470608c59ddbc66baf37c148e938c7038e6bc7a998aa"
],
"index": "pypi",
- "version": "==0.14.1"
+ "version": "==0.14.2"
},
"typing-extensions": {
"hashes": [
@@ -371,17 +372,18 @@
},
"uvloop": {
"hashes": [
- "sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd",
- "sha256:123ac9c0c7dd71464f58f1b4ee0bbd81285d96cdda8bc3519281b8973e3a461e",
- "sha256:4315d2ec3ca393dd5bc0b0089d23101276778c304d42faff5dc4579cb6caef09",
- "sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726",
- "sha256:afd5513c0ae414ec71d24f6f123614a80f3d27ca655a4fcf6cabe50994cc1891",
- "sha256:b4f591aa4b3fa7f32fb51e2ee9fea1b495eb75b0b3c8d0ca52514ad675ae63f7",
- "sha256:bcac356d62edd330080aed082e78d4b580ff260a677508718f88016333e2c9c5",
- "sha256:e7514d7a48c063226b7d06617cbb12a14278d4323a065a8d46a7962686ce2e95",
- "sha256:f07909cd9fc08c52d294b1570bba92186181ca01fe3dc9ffba68955273dd7362"
+ "sha256:1ae1ad731c8c0dcee80e0ecf06274f0f7293244d2cef81fa2747321a370a6aba",
+ "sha256:236a3c31096e0845029856f7bc07a938340c2cdb35d9d39b38c9253b672bf948",
+ "sha256:47ec567151070ed770211d359ad9250b59368548c60212c7ef6dda3f5b1778f6",
+ "sha256:66881fe8a2187334c4dd5010c56310bdf32fe426613f9ca727f090bc31280624",
+ "sha256:7846828112bfb49abc5fdfc47d0e4dfd7402115c9fde3c14c31818cfbeeb63dc",
+ "sha256:9541dc3f391941796ae95c9c3bb16b813acf9e3d4beebfd3b623f1acb22d318d",
+ "sha256:ca8a9e982f0bfbe331f41902cdd721c6e749e4685a403685e792b86a584f5969",
+ "sha256:e178c255622d928d464187e3ceba94db88465f6b17909c651483fb73af8d8b85",
+ "sha256:e72779681f839b6a069d7e7a9f7962a1d1927612c5c2e33071415478bdc1b91b",
+ "sha256:ed073d24e0c383c24d17d3a2bb209b999ff0a8130e89b7c3f033db9e0c3bd04f"
],
- "version": "==0.14.0"
+ "version": "==0.15.1"
},
"watchgod": {
"hashes": [
diff --git a/iamagesdb_create.py b/iamages_mkdb.py
similarity index 85%
rename from iamagesdb_create.py
rename to iamages_mkdb.py
index 717b04c..1e14ebe 100644
--- a/iamagesdb_create.py
+++ b/iamages_mkdb.py
@@ -1,10 +1,12 @@
-__version__ = "master"
+__version__ = "2.1.0"
__copyright__ = "© jkelol111 et al 2020-present"
import os
import json
import sqlite3
+SUPPORTED_FORMAT = 2
+
print("[Make Iamages Database version '{0}'. {1}]".format(__version__, __copyright__))
print("0/3: Load the server configuration file.")
@@ -28,4 +30,8 @@
with open("iamagesdb.sql", "r") as sqlscript:
storedb_cursor.executescript(sqlscript.read())
+server_config["files"]["storage"]["format"] = SUPPORTED_FORMAT
+
+json.dump(server_config, open("servercfg.json", "w"))
+
print("Done!")
\ No newline at end of file
diff --git a/iamages_replicator.py b/iamages_replicator.py
new file mode 100644
index 0000000..859f0f1
--- /dev/null
+++ b/iamages_replicator.py
@@ -0,0 +1,189 @@
+__version__ = "2.1.0"
+__copyright__ = "© jkelol111 et al 2021-present"
+
+import argparse
+import os
+import shutil
+import sqlite3
+import csv
+import json
+import hashlib
+import datetime
+import tempfile
+
+print("[Iamages Storage Replicator version '{0}'. {1}]".format(__version__, __copyright__))
+
+SUPPORTED_FORMAT = 2
+
+print("0/?: Load the server configuration file.")
+server_config = json.load(open("servercfg.json", "r"))
+
+print("0/?: Load the replicator configuration file.")
+replicator_config = json.load(open("replicatorcfg.json", "r"))
+
+print("0/?: Load previous replicated archives list.")
+archives = {}
+archives_filepath = os.path.join(replicator_config["directory"], "archives.json")
+if os.path.isfile(archives_filepath):
+ archives = json.load(open(archives_filepath, "r"))
+
+CMD_PARSER = argparse.ArgumentParser(
+ description='Back up or restore a replicated Iamages database.'
+)
+
+CMD_PARSER.add_argument(
+ 'command',
+ action='store',
+ help='The command to run (archive, restore, delete, list)')
+
+CMD_PARSER.add_argument(
+ 'archive_name',
+ action='store',
+ nargs="?",
+ help='The name of the replicated archive (for use with restore and delete command).')
+
+CMD_PARSED = CMD_PARSER.parse_args()
+
+def delete_archive(archive):
+ if not archive in archives:
+ raise FileNotFoundError("Archive {} not found!".format(archive))
+
+ os.remove(os.path.join(replicator_config["directory"], archive + ".zip"))
+ if archives[archive]["has_hash"]:
+ os.remove(os.path.join(replicator_config["directory"], archive + ".blake2b.txt"))
+ archives.pop(archive)
+ json.dump(archives, open(archives_filepath, "w"))
+
+if CMD_PARSED.command == "archive":
+ if not server_config["files"]["storage"]["format"] == SUPPORTED_FORMAT:
+ print(f'Current storage format is not supported. (expected: {SUPPORTED_FORMAT}, got: {server_config["files"]["format"]})')
+ exit(1)
+ with tempfile.TemporaryDirectory() as tmp:
+ shutil.copytree(server_config["files"]["storage"]["directory"], tmp, dirs_exist_ok=True, ignore=shutil.ignore_patterns("replicated"))
+
+ conn = sqlite3.connect(os.path.join(tmp, "iamages.db"))
+ cur = conn.cursor()
+ with open(os.path.join(tmp, "Files.csv"), "w") as csv_files:
+ writer = csv.writer(csv_files)
+ writer.writerow(["FileID", "FileName", "FileDescription", "FileNSFW", "FilePrivate", "FileMime", "FileWidth", "FileHeight", "FileHash", "FileLink", "FileCreatedDate", "FileExcludeSearch"])
+ writer.writerows(cur.execute("SELECT * FROM Files").fetchall())
+ with open(os.path.join(tmp, "Files_Users.csv"), "w") as csv_files_users:
+ writer = csv.writer(csv_files_users)
+ writer.writerow(["FileID", "UserID"])
+ writer.writerows(cur.execute("SELECT * FROM Files_Users").fetchall())
+ with open(os.path.join(tmp, "Users.csv"), "w") as csv_users:
+ writer = csv.writer(csv_users)
+ writer.writerow(["UserID", "UserName", "UserPassword", "UserBiography", "UserCreatedDate"])
+ writer.writerows(cur.execute("SELECT * FROM Users").fetchall())
+ conn.close()
+
+ os.remove(os.path.join(tmp, "iamages.db"))
+
+ current_datetime = datetime.datetime.now()
+ substitute_datetimes = {
+ "year": current_datetime.strftime("%Y"),
+ "month": current_datetime.strftime("%m"),
+ "day": current_datetime.strftime("%d"),
+ "hour": current_datetime.strftime("%H"),
+ "minute": current_datetime.strftime("%M"),
+ "second": current_datetime.strftime("%S")
+ }
+
+ replicated_filename = replicator_config["naming"].format(**substitute_datetimes) + ".iamagesbak"
+ replicated_filepath = os.path.join(replicator_config["directory"], replicated_filename)
+ shutil.make_archive(replicated_filepath, "zip", tmp)
+
+ if replicator_config["additional_options"]["save_archive_hash"]:
+ with open(replicated_filepath + ".zip", "rb") as replicated_file:
+ with open(os.path.join(replicator_config["directory"], replicated_filename + ".blake2b.txt"), "w") as replicated_file_hash:
+ replicated_file_hash.write(hashlib.blake2b(replicated_file.read()).hexdigest())
+
+ if len(archives) >= replicator_config["saves"]:
+ delete_archive(list(enumerate(archives))[-1][1])
+
+ archives[replicated_filename] = {
+ "format": replicator_config["format"],
+ "created_date": current_datetime.strftime("%Y/%m/%d %H:%M:%S"),
+ "has_hash": replicator_config["additional_options"]["save_archive_hash"]
+ }
+
+ json.dump(archives, open(archives_filepath, "w"))
+elif CMD_PARSED.command == "restore":
+ if not CMD_PARSED.archive_name:
+ print("Replicated name not provided. Exiting.")
+ exit(1)
+
+ replicated_info = archives[CMD_PARSED.archive_name]
+
+ if not replicated_info["format"] == SUPPORTED_FORMAT:
+ print(f'Replicated archive format is not supported. (expected: {SUPPORTED_FORMAT}, got: {replicated_info["format"]})')
+ exit(1)
+
+ replicated_filepath = os.path.join(os.getcwd(), replicator_config["directory"], CMD_PARSED.archive_name + ".zip")
+
+ with open(replicated_filepath, "rb") as replicated_archive:
+ with open(os.path.join(replicator_config["directory"], CMD_PARSED.archive_name.split(".zip")[0] + ".blake2b.txt"), "r") as replicated_archive_hash:
+ if not hashlib.blake2b(replicated_archive.read()).hexdigest() == replicated_archive_hash.read():
+ print("Replicated archive hash doesn't match saved hash in file. Exiting.")
+ exit(1)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ shutil.unpack_archive(replicated_filepath, tmp)
+
+ conn = sqlite3.connect(os.path.join(tmp, "iamages.db"))
+ cur = conn.cursor()
+ cur.execute("PRAGMA journal_mode=WAL")
+
+ FILES_CSV_PATH = os.path.join(tmp, "Files.csv")
+ with open(FILES_CSV_PATH, "r") as csv_files:
+ cur.execute("CREATE TABLE Files (FileID INTEGER PRIMARY KEY, FileName TEXT, FileDescription TEXT, FileNSFW INTEGER, FilePrivate INTEGER, FileMime TEXT, FileWidth INTEGER, FileHeight INTEGER, FileHash TEXT, FileLink INTEGER, FileCreatedDate TEXT, FileExcludeSearch INTEGER)")
+ reader = csv.DictReader(csv_files)
+ for row in reader:
+ cur.execute("INSERT INTO Files (FileID, FileName, FileDescription, FileNSFW, FilePrivate, FileMime, FileWidth, FileHeight, FileHash, FileLink, FileCreatedDate, FileExcludeSearch) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+ row["FileID"], row["FileName"], row["FileDescription"], row["FileNSFW"], row["FilePrivate"], row["FileMime"], row["FileWidth"], row["FileHeight"], row["FileHash"], row["FileLink"], row["FileCreatedDate"], row["FileExcludeSearch"]))
+ os.remove(FILES_CSV_PATH)
+
+ FILES_USERS_CSV_PATH = os.path.join(tmp, "Files_Users.csv")
+ with open(FILES_USERS_CSV_PATH, "r") as csv_files_users:
+ cur.execute("CREATE TABLE Files_Users (FileID INTEGER, UserID INTEGER)")
+ reader = csv.DictReader(csv_files_users)
+ for row in reader:
+ cur.execute("INSERT INTO Files_Users (FileID, UserID) VALUES (?, ?)", (
+ row["FileID"], row["UserID"]))
+ os.remove(FILES_USERS_CSV_PATH)
+
+ USERS_CSV_PATH = os.path.join(tmp, "Users.csv")
+ with open(USERS_CSV_PATH, "r") as csv_users:
+ cur.execute("CREATE TABLE Users (UserID INTEGER PRIMARY KEY, UserName TEXT, UserPassword TEXT, UserBiography TEXT, UserCreatedDate TEXT)")
+ reader = csv.DictReader(csv_users)
+ for row in reader:
+ cur.execute("INSERT INTO Users (UserID, UserName, UserPassword, UserBiography, UserCreatedDate) VALUES (?, ?, ?, ?, ?)", (
+ row["UserID"], row["UserName"], row["UserPassword"], row["UserBiography"], row["UserCreatedDate"]))
+ os.remove(USERS_CSV_PATH)
+
+ conn.commit()
+ conn.close()
+
+ shutil.rmtree(server_config["files"]["storage"]["directory"])
+ shutil.copytree(tmp, server_config["files"]["storage"]["directory"])
+elif CMD_PARSED.command == "delete":
+ delete_archive(CMD_PARSED.archive_name)
+elif CMD_PARSED.command == "list":
+ print("\nAvailable replicated archives:\n")
+ for save in archives:
+ archive_filepath = os.path.join(replicator_config["directory"], save + ".zip")
+ if os.path.isfile(archive_filepath):
+ print(f'- {save} ({archive_filepath})')
+ print(f' + Created date: {archives[save]["created_date"]}')
+ print(f' + Archive format: {archives[save]["format"]}')
+ if archives[save]["has_hash"]:
+ archive_hash_filepath = os.path.join(replicator_config["directory"], save + ".blake2b.txt")
+ if os.path.isfile(archive_hash_filepath):
+ print(f' + Archive hash: enabled ({archive_hash_filepath})')
+ else:
+ print(" + Archive hash: enabled (not found)")
+ else:
+ print(f' + Archive hash: disabled')
+ else:
+ print(f'- {save} (not found)')
+ print("")
diff --git a/iamages_server.py b/iamages_server.py
index 2dbeec5..848bc5d 100644
--- a/iamages_server.py
+++ b/iamages_server.py
@@ -1,4 +1,4 @@
-__version__ = "2.0.0"
+__version__ = "2.1.0"
__copyright__ = "© jkelol111 et al 2020-present"
import os
@@ -28,6 +28,12 @@
server_config = json.load(open("servercfg.json", "r"))
+SUPPORTED_FORMAT = 2
+
+if server_config["files"]["storage"]["format"] != SUPPORTED_FORMAT:
+ print(f'Current storage format is not supported. (expected: {SUPPORTED_FORMAT}, got: {server_config["files"]["format"]})')
+ exit(1)
+
if not os.path.isdir(server_config["files"]["storage"]["directory"]):
os.makedirs(server_config["files"]["storage"]["directory"])
@@ -186,7 +192,7 @@ async def get(self, request):
response_body = {
"FileIDs": []
}
- FileIDs = await iamagesdb.fetch_all("SELECT FileID FROM Files WHERE FilePrivate = 0 ORDER BY FileID DESC LIMIT 10")
+ FileIDs = await iamagesdb.fetch_all("SELECT FileID FROM Files WHERE FilePrivate = 0 AND FileExcludeSearch = 0 ORDER BY FileID DESC LIMIT 10")
for FileID in FileIDs:
response_body["FileIDs"].append(FileID[0])
return starlette.responses.JSONResponse(response_body)
@@ -200,7 +206,7 @@ async def get(self, request):
attempts = 0
while successful_FileID == 0 and attempts <= 3:
successful_FileID = random.randint(1, total_files)
- actual_successful_FileID = await iamagesdb.fetch_one("SELECT FileID From Files WHERE FileID = :FileID AND FilePrivate = 0", {
+ actual_successful_FileID = await iamagesdb.fetch_one("SELECT FileID From Files WHERE FileID = :FileID AND FilePrivate = 0 AND FileExcludeSearch = 0", {
"FileID": successful_FileID
})
if not actual_successful_FileID:
@@ -238,6 +244,7 @@ async def put(self, request):
"FileHash": FileHash
})
default_query_FilePrivate = "UPDATE Files SET FilePrivate = :FilePrivate WHERE FileID = " + str(FileID)
+ default_query_FileExcludeSearch = "UPDATE Files SET FileExcludeSearch = :FileExcludeSearch WHERE FileID = " + str(FileID)
if duplicate_exists:
await iamagesdb.execute("UPDATE Files SET FileLink = :FileLink WHERE FileID = :FileID", {
"FileLink": duplicate_exists[0][0],
@@ -260,6 +267,14 @@ async def put(self, request):
await iamagesdb.execute(default_query_FilePrivate, {
"FilePrivate": False
})
+ if "FileExcludeSearch" in request_body:
+ await iamagesdb.execute(default_query_FileExcludeSearch, {
+ "FileExcludeSearch": request_body["FileExcludeSearch"]
+ })
+ else:
+ await iamagesdb.execute(default_query_FileExcludeSearch, {
+ "FileExcludeSearch": False
+ })
response_body["FileID"] = FileID
return starlette.responses.JSONResponse(response_body)
else:
@@ -304,6 +319,14 @@ async def put(self, request):
await iamagesdb.execute(default_query_FilePrivate, {
"FilePrivate": False
})
+ if "FileExcludeSearch" in request_body:
+ await iamagesdb.execute(default_query_FileExcludeSearch, {
+ "FileExcludeSearch": request_body["FileExcludeSearch"]
+ })
+ else:
+ await iamagesdb.execute(default_query_FileExcludeSearch, {
+ "FileExcludeSearch": False
+ })
else:
await SharedFunctions.delete_file(FileID)
return starlette.responses.Response(status_code=415)
@@ -330,7 +353,7 @@ async def patch(self, request):
if FileID == request_body["FileID"]:
basic_query = "UPDATE Files SET {0} = :value WHERE FileID = " + str(FileID)
for modification in request_body["Modifications"]:
- if modification in ["FileDescription", "FileNSFW", "FilePrivate"]:
+ if modification in ["FileDescription", "FileNSFW", "FilePrivate", "FileExcludeSearch"]:
basic_query = basic_query.format(modification)
await iamagesdb.execute(basic_query, {
"value": request_body["Modifications"][modification]
@@ -358,7 +381,8 @@ async def get(self, request):
"FileMime": None,
"FileWidth": None,
"FileHeight": None,
- "FileCreatedDate": None
+ "FileCreatedDate": None,
+ "FileExcludeSearch": None
}
async def set_response(FileInformation):
response_body["FileID"] = int(request.path_params["FileID"])
@@ -377,9 +401,10 @@ async def set_response(FileInformation):
response_body["FileWidth"] = FileInformation[4]
response_body["FileHeight"] = FileInformation[5]
response_body["FileCreatedDate"] = FileInformation[6]
+ response_body["FileExcludeSearch"] = bool(FileInformation[8])
FileID = int(request.path_params["FileID"])
- FileInformation = await iamagesdb.fetch_one("SELECT FileDescription, FileNSFW, FilePrivate, FileMime, FileWidth, FileHeight, FileCreatedDate, FileLink FROM Files WHERE FileID = :FileID", {
+ FileInformation = await iamagesdb.fetch_one("SELECT FileDescription, FileNSFW, FilePrivate, FileMime, FileWidth, FileHeight, FileCreatedDate, FileLink, FileExcludeSearch FROM Files WHERE FileID = :FileID", {
"FileID": FileID
})
FilePrivate = bool(FileInformation[2])
diff --git a/iamages_updb.py b/iamages_updb.py
new file mode 100644
index 0000000..4efeb4c
--- /dev/null
+++ b/iamages_updb.py
@@ -0,0 +1,54 @@
+__version__ = "2.1.0"
+__copyright__ = "© jkelol111 et al 2020-present"
+
+import os
+import json
+import sqlite3
+
+print("[Upgrade Iamages Database version '{0}'. {1}]".format(__version__, __copyright__))
+
+BASE_FORMAT = 1
+UPGRADED_FORMAT = 2
+
+print("0/3: Load the server configuration file.")
+server_config = json.load(open("servercfg.json", "r"))
+
+if not server_config["files"]["storage"]["format"] == BASE_FORMAT:
+ print("This script doesn't upgrade from this database version! Exiting. (got: {}, expected: {})".format(server_config["files"]["storage"]["format"], BASE_FORMAT))
+ exit(1)
+
+print("1/3: Analysing existing database.")
+FILESDB_PATH = os.path.join(server_config["files"]["storage"]["directory"], "iamages.db")
+
+if not os.path.isfile(FILESDB_PATH):
+ print("Database doesn't exist! Exiting.")
+ exit(1)
+
+storedb_connection = sqlite3.connect(FILESDB_PATH)
+storedb_cursor = storedb_connection.cursor()
+
+files_table_columns = storedb_cursor.execute("PRAGMA table_info('Files')").fetchall()
+
+FilesExcludeSearch_found = False
+
+for files_table_column in files_table_columns:
+ if files_table_column[1] == "FileExcludeSearch":
+ FilesExcludeSearch_found = True
+ print("FilesExcludeSearch column found. No change required.")
+ break
+
+if not FilesExcludeSearch_found:
+ print("2/3: Performing database upgrade.")
+ storedb_cursor.execute("ALTER TABLE Files ADD FileExcludeSearch INTEGER")
+ FileIDs = storedb_cursor.execute("SELECT FileID FROM Files").fetchall()
+ for FileID in FileIDs:
+ storedb_cursor.execute("UPDATE Files SET FileExcludeSearch = ? WHERE FileID = ?", (False, FileID[0]))
+
+storedb_connection.commit()
+storedb_connection.close()
+
+print("3/3: Updating server configuration.")
+server_config["files"]["storage"]["format"] = UPGRADED_FORMAT
+json.dump(server_config, open("servercfg.json", "w"))
+
+print("Done!")
\ No newline at end of file
diff --git a/iamagesdb.sql b/iamagesdb.sql
index f779c7b..9a6a6b2 100644
--- a/iamagesdb.sql
+++ b/iamagesdb.sql
@@ -9,12 +9,15 @@ CREATE TABLE Files (
FileHeight INTEGER,
FileHash TEXT,
FileLink INTEGER,
- FileCreatedDate TEXT
+ FileCreatedDate TEXT,
+ FileExcludeSearch INTEGER
);
+
CREATE TABLE Files_Users (
FileID INTEGER,
UserID INTEGER
);
+
CREATE TABLE Users (
UserID INTEGER PRIMARY KEY,
UserName TEXT,
diff --git a/requirements.txt b/requirements.txt
index fb27bbb..b24ee4f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@
aiofiles==0.6.0
aiosqlite==0.16.1
bcrypt==3.2.0
-cffi==1.14.4
+cffi==1.14.5
click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
databases[sqlite]==0.4.1
filetype==1.0.7
@@ -23,10 +23,10 @@ pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.
python-dotenv==0.15.0
pyyaml==5.4.1
six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sqlalchemy==1.3.22; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-starlette==0.14.1
+sqlalchemy==1.3.23; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+starlette==0.14.2
typing-extensions==3.7.4.3
uvicorn[standard]==0.13.3
-uvloop==0.14.0
+uvloop==0.15.1
watchgod==0.6
websockets==8.1
diff --git a/templates/tos.html b/templates/tos.html
index 005314d..ebf0175 100644
--- a/templates/tos.html
+++ b/templates/tos.html
@@ -22,12 +22,15 @@ Terms of Service
We (The Developers) assume NO responsibility when you decide to use the Service.
- We deny serving any 'illegal' content (Illegal Content), such as, but not limited to:
+ We deny serving any 'illegal' Content (Illegal Content), such as, but not limited to:
- Overly sexual content (especially those involving minors)
- Overly violent content
- Promotes illegal activities
+
+ Moreover, other Content prohibited by our hosting provider (Illegal Content), Uberspace is not allowed on our Service.
+ To know more, please refer to their house rules.
If any such Illegal Content is to be found on our systems, please report it to