From d0ac89b622702382efcc376a5baaebb49137f4c2 Mon Sep 17 00:00:00 2001 From: Dominik Roth Date: Tue, 9 Jun 2020 21:50:18 +0200 Subject: [PATCH] everything works! (except for compression) --- __pycache__/main.cpython-38.pyc | Bin 0 -> 13314 bytes main.py | 226 +++++++++++++++++++++----------- 2 files changed, 149 insertions(+), 77 deletions(-) create mode 100644 __pycache__/main.cpython-38.pyc diff --git a/__pycache__/main.cpython-38.pyc b/__pycache__/main.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fc8791792ca66f422ed570644748f48b12cfed5 GIT binary patch literal 13314 zcmbtbYm6k{@Z+fB!JnkhfkbbYs(_R}`&oJOXZ^|O*sHN0ld&q>~Gc&$n&6oP@aeUL-IWAAC~74 z{|KJB#?j_6|Co%CZyawvBlyI{I(~e&?j# zbh=i>Ze_YQpUT3Z?(E%@*P>`GeD2Jd7AP1_Mc0C;+FEV|Q*FKc+%wO-d*Qb2=Nj!= zwNY7X-z+Gc!Gj&_hn3)_q^d@eBRY zpWQ5Wa~P|*cVroLsGHx;M}=-dIZ;ui&}&G!Z@Lt*>!0gpw)0(UWoTu%o9`BuZIyo0 zQJEu{Giu+}&0rI=vv;$vRv|#Imo&)|XD(k^I6pHF{?76>le%$!Nw=GoMZV%@wXma=N}NWwgziJSH4S_R*&t4i%!P+JYxn{lSPwidLMrtD~Lw=j=Cx~7A* zstzi>`ILwCA%5OfRbYn4Wn7K|?R)sj)j%(7_=RW#nG5SJ)xf%%A=b4hh0ymnOM*sh zFZwz2_N8#SmtEKhW0xbwcI2m{4O9v5n&b#STc|7tEi-obC|-o`MPk``+XZ)?uuFKl z_9XI0?Ze19@-JgM-*xY0Jl96fcE9JQod0s}Wip-P_w(}Z@B=bTl2ca+6VD<8hI1lF zvM}h`*PPANrW@Ij6Q#OfWWGNJ>7E1wZb8QTjF<+?WLC0W7g7%q0IL+cj>!8Rg;DR? zkCkoBg_XTH7qr%!guFmMg*W=0Na7TbQa{P!6cb9gUkI^t*dbw$wCs<5niY9N;<*6q zW7p%1Dek9fvBLfdQO}`YIF7^u`9_7HAlf~UFWq_Qesb;UStIN8{z$Jr=AA`>SVv)u zY!P=*X*<<*NCC+SEs%2o1#Z{cP6G?w9S5~8sN>9biZ8D<+EoS2pPHH~XM`eg_F6T( z)~GMWUU;qg?#jF0`%Ii!Ujt_baYogbgD{GR)}`MiaO+(4W;vnG8O$iogh5RQQEWHk zboc>`w4X5f-M$}0hmc4{6!gGvd89?VbL>8n?DiQT2%vG7f!;ttP|$$_5H#DtQ#pb< zsmLK|P_|i)oD~zzPTZZy{)kAU9ToJt)mO?yE<%PQD~ zV0Xi|+!#I0wtzdD5^<(hU%M9Q*sDbb?ex3YhWx#6zw`yXTfi?YBC!AgPuS!3L2lED z|6!ZJ76Us)w(YYnY0}qMDNxUWrf_y3vbXKoa>~eL21|#I)-z~ab`3tJqK!xsEwn6& zD%ea{U@T|2WMLzE4HY3jM-b~AeVcir;Q$5lqYKb2d&LXp1-OU2BV3gdt^zP^ z1D1MR7Nu4oV&IWPmm1y!SEZnV9q><<+@rGKA6QA7>8=Ys+}+IJJ2~jr%z%EiR`M$a zXvqAQy=-?g(1iu)!Xor?HX7zdhW==rX$@wDGzg#y+u`>*iKQ zkQ;|42Pb#)E2CTXd#ug$^>1DO=5^IgD-Wams~VmMh;-B2V`_xv$ zT93F^G_jh8sr;mE@gKDJ$RW$r|ESe+K4L|a0Gb2c;*Jd}9Ncm?hq^-mnbFzK@TK<6 z)-KHG7eHeD0VbE3P=||b=pR8Yw)8`MK?8{v7c_Y2=MudhgnowL6okU5|Bg0RY`?F3 z$Ur&aaaWY|ODs~W7+JZ>9MN8%W+E6rwp164@0;eK(b7}}eqlL)a3o5$D5~1n@1cJ* ztS`5!(K=LkVqH|0balBIw4ykpVGh*Se6JcNqLt(KC_S)&0ge>sD<~yOUj$PVEA$)_ z3Zn0c-l3{~hQ&;@Ay6vvN#OULXd`I^??KPAR48E2Ku(q}QXR)9-~v z{W=;zlZX#<+c5}5aM}kvaR#4)h~7d~$S+uS=g@;EN5f44I4Igp=6YR zv9S_T*MbeRmFb%rst48sO;|za*j|e54Wn`FTl*>H!nlq;g|><6FM`pEcG1aGryRI% zhuOLewczMu9RE0y@{kzk)n)+WJYT6a+iJbRe6dowzFuv#q(&cQ`@S+XbHBiMuP`AS z>W?w0GFfG^yQh+)R?!nk2ua~266|Ouf7~m0InVQ4?}(T3GSZ@k%7H&};5q!l4J3Vc z`9$AcK1o#|?s8UnaC_4#r}B7aOauc^A+Za%%M!JKyF8}GQJ+^6Y7)-^qYii$)gg5l z&mlPANAVn1$JB8=N7O@*yAr%3aX}aE6A_Auu-Z=|V_2={G)HiqSD*#pI72TZb))ol zW(O=+$to9uWUaXRY&WIS&?GEnw>=0Kyh9NwQ?`5MI{W1nET&39!B(mimB%{dhNNy7 ztUR&b8_KYJuKlrcL?69DNAf9Mc!wB(-+A88D7@-lsMm}Ee=cQ^c2y4~WLcJNZ z*Q0p2b`7zBOF^UFq`elWYgKrQ@o;4+h-%knl97A%PC9C@1}&}eT^~>`tKY+?4mf4I%ah6;i;=5&OY5LN+$bI9+TZ%nn?Tjw_FlQ0O?QD;1v-<~El+6TQq z?rWcK{|$v*3vJ8Vv{8bU1yHwEVC@6)F=NgNVTd7CPPU(TKPhTX*2`g$JBG{|1eDB`9do!JP?i=oA74q5_fgdXAKzbV= zbK$P{V{?~hU)X%E)OqClf3AGcD4VoIWORt>i%JIkgGDg?MUjozUGCJ^;yi5%BgZ1I zu?G)4WL!qyL{E5LYmIuO*O_-C2qfM#z_$7Z$0aix&m)WQ%38Iy8mDQshQiT$2-S(B z3!x=RjebinQvUsHZC%e{jr+J=M*T9nhSWo}2r`bb3Miv=dTDH#oGWIRhx!b19wP+~ zpFN#8HHZ-dfKJa{4&he4!*rRsVkibr5Q>*b)wa$#aaAW8*vJ#QWt;?!zz zOP8>w*k#amKZSn=HGha-IEutdkJ@=qZP{L%pK5mEZ6Lh}vsX6bPv z9?e&fhB6D{;61`8kWJBtQsTSpAwilI(Q5GZ@`!8}=vKPm`7BqEBAY>$$7 z5{j|VZ_JpbqW@B}#vA%LCQ=?3Zq+}Iy#5I$T&-cq&mb44)p~O+6!zYg4&x1pt3%D9 zX=}mL?}^5Bz7sp>9CC@O@e2q%^ftoJq9EKn&9=K%5fx56yuKC&x{u-C#yhc%Eb^^0 z>WqV*sde(uMSb2K>@z@qsvYvL@Gez92qeL5hHSV23=Dw3?wms%rYrNGK|ao!SW}4T zDD|Xqw|ctgLGF-OLC}}5@f~#UITd}SnA8nSj{1;4nH)K+H#yl!Ow8y)$IOa0CE^#N z`-F54n1uc$8vPS~Nq_=IZ+`{}s13|ufIH8jA9zumD;o8p*KB=4qFaexL&V?Ru}r!w zdarAr<-8G_IMR1@GBOW=Phk+fg`o|j(PHg6JzPcXEqjz&;YsHy22N*q(4ZN>Z2E|4{DifB3mGVU8$lSeNJO{0 zy$H84K^|8KB(oyZW*r-|6Q|Le{(X znL98F?v?Wo!XW)~Sfu_%CPHK(?WbA%c_h7H}(rXb;mH=K+R`_$9S40vx>!O1cg3CO4;%yXT|o3-MQ`D`+m7B{aS}J*VdWQ@yv5{8 zO!o0vy1Af? zdZrIAyR8RgjN8{=KSKw7@B}bolelTlPRf||Jt+SQ+UYMN>A7L4c0-wP;)7=DG;ch{ z8S+~aZ_f3$;|31m7s~bw{Lco*Y%*yvIm%>{iI|~ zqdBl9+0xcCNm-2VP3QV?F}2f)g(#r%5qxC^j7kM!IO@~L#X~?zV&>(LXxP){J8bk8 zlP@tDTyLC*GMhKuPIK8tG5vJ+FgT?XWiGRL3n$s95y1hrG22vG04 z2Pq)NQ69Z@Qy{|A$)bC3TB4~8#4$bKg`732e1Htw_{I#5_&u5`R1X5Py^~3<+t0G< z%Hqn~6JW@kq>*B1_de2~h@LcTPf&>vaC$d2$cIw{?FTr4#zdbJc(6pKUGIo~7Ud(Q z%=sFeQA$A>q3Ob_r(Q2zoSR0#!B`l^H^LlfXN#34j8BNXofzIb$m6cY{jMTTJ>{hI z1ulUTvfyVQbb3)I!_cP5?7#vFXxiuI@1iicfaKy0H42?NJA8uIZno@C*-?5s&9is0 zzO3yGYxOJ)+w6*W!NTDiEVI_IlykkZfgH)`AI9?-&K1ya6eCilari(s2v-N9N~=+Ct>QjF zR9Ua%#2L@!x|)%OJi(SWxOs*!3|lvu6B9{_xG2{Q>LB&Gx&^L{b7&%w7GBkneg7&d z_3tvFTc#;v#;l}#@OZr%#o77F<(awZ7fj~T<+<2t zzitW|beI~Mc%^O z*ls+~c*t^f@KHWNIvC4gm(R)5h;t0^angQ(AmrmdK{$x9pJ>i8R>qs5&{cvGFysXJ zX$|)*x!cGaJAH72a3&w{Oz^Xu8^`A{$sp{T2KOmUPUs?j5?t&}h9N;tMgJ9Qdh?)Z zj(ne^_VEbr5H>-b&+pCv^hjbOMqH$R;EpHN4p~GdKuh8OEiMjkf$)weU4H^R1qc`G zaK{8-FG7S^Ca=@sybpB)IvHP=lM>V3=x4!?SLQBLJ^v|YC?L`qdxa;$$q9lSA@msT z&)8-cub*VGufX@huQ_f1bJq8TQU3*t(@grJ7^igg<~~~MWiUVLeJrpWo$6J%QUnO`Kk8;F8Y%87NeJl`}}J)Bz;q zH~+8SGyJemz=1xEpc)2W+w;YJXe0IFkI$AT9_WwDHvA6eEhe_HEq*`1!`#GU+2RFpZt5;EA2lvl(hf)xJc^N)CGnk~)~ek!Hr!|U zI$I5IwQBXY{xO!)f!3|I*m=IY1n*0aZ~zgTG3KZ*^f#IO0h3QL`2{8uOz8CsQDy6K z$6DizPlq^qKd+Z@Be)e^Wl`gQoG{NKW?exmsf+XSmSx_iMk+g^|CXaZV6$WIthto( zJn0_3h=ez-c+CZOt00HQ&ZC7#-RR*h!;~gJ+#de8bz=PER?!>`%Dq^}L2YvP`al00 D20?cq literal 0 HcmV?d00001 diff --git a/main.py b/main.py index 04f7811..afddfba 100644 --- a/main.py +++ b/main.py @@ -23,6 +23,8 @@ from errno import ENOENT from fuse import FUSE, FuseOSError, Operations, LoggingMixIn import stat import os +import gzip +import secrets class IotaFS_BlobStore(): def __init__(self, api=None): @@ -31,7 +33,7 @@ class IotaFS_BlobStore(): else: self.api = api - def genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1): + def _genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1): msg = TryteString.from_bytes(data) bundles = [] nextAddr = addrIter.__next__() @@ -59,7 +61,7 @@ class IotaFS_BlobStore(): ) return bundles - def sendBundles(self, bundles): + def _sendBundles(self, bundles): bundleRets = [] for i,bundle in enumerate(bundles): print(str(int(i/len(bundles)*100))+"%") @@ -84,8 +86,8 @@ class IotaFS_BlobStore(): cipher = AES.new(sHash[:16], AES.MODE_CBC, sHash[22:][:16]) ct_bytes = cipher.encrypt(pad(data, AES.block_size)) addrIter = AddressGenerator(Seed(trSeed)).create_iterator(start = 0, step = 1) - bundles = self.genBundles(ct_bytes, addrIter) - self.sendBundles(bundles) + bundles = self._genBundles(ct_bytes, addrIter) + self._sendBundles(bundles) def uploadTxt(self, txt, secret): data = str.encode(txt) @@ -107,7 +109,7 @@ class IotaFS_BlobStore(): for tx in bundle.transactions: tryteMsg+=str(tx.signature_message_fragment) if tryteMsg == "": - return "" + return b'' tryteStr = TryteString(tryteMsg.rstrip("9")) try: ct_bytes = tryteStr.as_bytes() @@ -136,16 +138,22 @@ class IotaFS_BlobStore(): f.write(y) class IotaFS(): - def __init__(self, token): + def __init__(self, token, fileCompression=False): self.api = Iota('https://nodes.thetangle.org:443', local_pow=True) self.blobStore = IotaFS_BlobStore(self.api) #self.token = token + self.fileCompression = fileCompression self.hashState = hashlib.sha3_384() + genesis = "This is the genesis block. lol." + if self.fileCompression: + raise Exception("Compression does not work currently") + genesis += "#FILE COMPRESSION#" + self.hashState.update(genesis.encode()) self.hashState.update(token.encode()) self._fileTree = {} self.lastBlockIncomplete = False self.incompleteBlockRescanTimeout = 5 - self.chainDelimiter = "#CHAIN_DELIM#" + self.chainDelimiter = "#IOTA_FS#".encode() self.cache = {} self._fetchFileTree() @@ -155,34 +163,34 @@ class IotaFS(): self._fetchFileTree() return copy.deepcopy(self._fileTree) - async def fileTreeFetchLoop(self, interval=10): - while True: - self._fetchFileTree() - await asyncio.sleep(10) - def _fetchFileTree(self): print("[<] Fetching FileTree") - chain = "" + chain = bytes() while True: print("[<] Fetching FileTree-ChainBlock") sHash = self.hashState.digest() block = self._getBlob(sHash) - if block=="": + data = block + if self.fileCompression: + data = gzip.decompress(data) + if data==b'': print("[-] Last Block Received") break self.hashState.update(block) - chain+=block - if chain=="": - print("[.] FileTree succesfully fetched: [EMPTY FILE TREE]") + chain+=data + if chain==b'': + print("[.] FileTree succesfully fetched: [NO UPDATES]") return if chain.endswith(self.chainDelimiter): - curRing = chain.split(self.chainDelimiter)[-1] + curRing = chain.split(self.chainDelimiter)[-2] self.lastBlockIncomplete = False else: print("[-] Last Block was incomplete; refetching...") self.lastBlockIncomplete = True time.sleep(self.incompleteBlockRescanTimeout) self._fetchFileTree() + return + print("{RING}: "+str(curRing)) self._fileTree = msgpack.loads(curRing) print("[.] FileTree succesfully fetched: ") @@ -200,7 +208,7 @@ class IotaFS(): self._mergeFileTrees(value, node) else: if key in treeA and treeA[key]=="#REMOVE#" or key in treeB and treeB[key]=="#REMOVE#": - treeB[key] = "#REMOVE#" + del treeB[key] treeB[key] = value return treeB @@ -213,71 +221,117 @@ class IotaFS(): newRing = msgpack.dumps(self._fileTree)+self.chainDelimiter sHash = self.hashState.digest() - self.blobStore.uploadDataRaw(newRing.encode(), sHash) + print("{RING}: "+str(newRing)) + payload = newRing + if self.fileCompression: + payload = gzip.compress(payload) + self.blobStore.uploadDataRaw(payload, sHash) self.hashState.update(newRing) # For every link in the chain, we salt our hashState using the links data def _putBlob(self, data): - #TODO: Use secure random provider - return self.blobStore.uploadData(data, str(random.random()*999999999999).encode()) + return self.blobStore.uploadData(data, secrets.token_bytes(64)) def _getBlob(self, sHash): - return self.blobStore.getData(sHash) + data = self.blobStore.getData(sHash) + return data def _fetchFile(self, sHash): file = self._getBlob(sHash) - # file lastFetch lastAccess - self.cache[sHash] = [file, time.now(), time.now()] + blob = file + if self.fileCompression: + blob = gzip.decompress(blob) + # file lastFetch lastAccess + self.cache[sHash] = [blob, time.time(), time.time()] return self.cache[sHash] def getFile(self, sHash): + if sHash==b'': + return [b'', 0, time.time()] + print("/GET/ "+str(sHash)+" <- ") if sHash in self.cache: # TODO: maybe update if to old? - self.cache[sHash][2] = time.now() + self.cache[sHash][2] = time.time() return self.cache[sHash] else: return self._fetchFile(sHash) def putFile(self, file, path): - blablabla + print("/PUT/ "+str(file)+" -> "+path) + if file==b'': + sHash = b'' + else: + blob = file + if self.fileCompression: + blob = gzip.compress(blob) + sHash = self._putBlob(blob) + self.cache[sHash] = [file, time.time(), time.time()] + treeDelta = {} + subTree = treeDelta + for elem in path.split("/")[:-1]: + subTree[elem] = {} + subTree = subTree[elem] + subTree[path.split("/")[-1]] = sHash + self.upsertFileTree(treeDelta) + def mkdir(self, path): + treeDelta = {} + subTree = treeDelta + for elem in path.split("/"): + subTree[elem] = {} + subTree = subTree[elem] + self.upsertFileTree(treeDelta) + return 0 + + def removeFile(self, path): + treeDelta = {} + subTree = treeDelta + for elem in path.split("/")[:-1]: + subTree[elem] = {} + subTree = subTree[elem] + file = subTree[path.split("/")[-1]] + subTree[path.split("/")[-1]] = "#REMOVE#" + self.upsertFileTree(treeDelta) + return file + + def mv(self, old, new): + file = self.removeFile(old) + self.putFile(self, file, new) class IotaFS_Fuse(LoggingMixIn, Operations): - def __init__(self, token): - self.fs = IotaFS(token) - self.fileTree = self.fs.getFileTree() + def __init__(self, token, fileCompression=True): + self.fs = IotaFS(token, fileCompression=fileCompression) def getSubtree(self, path): - subTree = self.fileTree - for elem in path.split("/"): - if elem not in subTree: - return False - subTree = subTree[elem] + subTree = self.fs.getFileTree() + for elem in path[1:].split("/"): + if elem!="": + if elem not in subTree: + return False + if self.subtreeIsFile(subTree): + # we cannot traverse further, if this is a file + return False + subTree = subTree[elem] return subTree def createFileObj(self, path, fileObj): - for elem in path.split("/")[:-1]: + subTree = self.fileTree + for elem in path[1:].split("/")[:-1]: if elem not in subTree: return False subTree = subTree[elem] subTree[path.split("/")[-1]] = fileObj def subtreeIsFile(self, subtree): - return isinstance(subtree, str) + return isinstance(subtree, (bytes, bytearray)) def subtreeExists(self, subtree): return not (subtree == False) - #def chmod(self, path, mode): - # return self.sftp.chmod(path, mode) - - #def chown(self, path, uid, gid): - # return self.sftp.chown(path, uid, gid) - - #def create(self, path, mode): - # f = self.sftp.open(path, 'w') - # f.chmod(mode) - # f.close() - # return 0 + def create(self, path, mode): + print("[#] CREATE "+path) + self.fs.putFile(b'', path[1:]) + #return open(path[1:]) + return 0 #def destroy(self, path): # self.sftp.close() @@ -293,69 +347,81 @@ class IotaFS_Fuse(LoggingMixIn, Operations): st = {} # mode decides access permissions and if file object is a directory (stat.S_IFDIR), file (stat.S_IFREG) or a special file if self.subtreeIsFile(subTree): - st['st_mode'] = 777 | stat.S_IFREG + st['st_mode'] = 0o744 | stat.S_IFREG else: - st['st_mode'] = 777 | stat.S_IFDIR - st['st_ino'] = 0 - st['st_dev'] = 0 + st['st_mode'] = 0o744 | stat.S_IFDIR + #st['st_ino'] = 0 + #st['st_dev'] = 0 st['st_nlink'] = 1 st['st_uid'] = os.getuid() #file object's user id st['st_gid'] = os.getgid() #file object's group id - if fh: + if fh and False: file, path, sHash, lastFetch, lastAccess = fh st["st_size"] = len(file) st['st_atime'] = lastAccess st['st_mtime'] = lastFetch st['st_ctime'] = 0 else: - st['st_size'] = 0 # 0 Byte lol - st['st_atime'] = now #last access time in seconds - st['st_mtime'] = now #last modified time in seconds + st['st_size'] = 1024*1024*1024 # 1 Byte lol + st['st_atime'] = 0 #last access time in seconds + st['st_mtime'] = 0 #last modified time in seconds st['st_ctime'] = 0 # very old file # TODO: Actuall real value block_size = 512 st['st_blocks'] = (int) ((st['st_size'] + block_size-1) / block_size) return st - #def mkdir(self, path, mode): - # return self.sftp.mkdir(path, mode) + def mkdir(self, path, mode): + print("[#] MKDIR "+path) + self.fs.mkdir(path[1:]) + return 0 - def read(self, path2, size, offset, fh): - print("[#] WRITE "+path2) - file, path, sHash, lastFetch, lastAccess = fh + def read(self, path, size, offset, fh): + print("[#] READ "+path) + file, path2, sHash, lastFetch, lastAccess = self.openFile(path) if path!=path2: + print(path+"!="+path2) return FuseOSError(ENOENT) - return file[offset : offset+size] def readdir(self, path, fh): print("[#] READDIR "+path) - l = ['.', '..'] - for elem in self.getSubtree(path): - l.append(elem.encode('utf-8')) + subTree = self.getSubtree(path) + if self.subtreeIsFile(subTree): + # We cant list a file! + return FuseOSError(ENOENT) + pprint(subTree) + l = [".", ".."] + for elem in subTree: + l.append(elem) return l - #def rename(self, old, new): - # return self.sftp.rename(old, new) + def rename(self, old, new): + self.fs.mv(old,new) + return 0 - #def rmdir(self, path): - # return self.sftp.rmdir(path) + def rmdir(self, path): + self.fs.removeFile(path) + return 0 - def write(self, path2, data, offset, fh): - print("[#] WRITE "+path2) - file, path, sHash, lastFetch, lastAccess = fh + def write(self, path, data, offset, fh): + print("[#] WRITE "+path) + file, path2, sHash, lastFetch, lastAccess = self.openFile(path) if path!=path2: + print(path+"!="+path2) return FuseOSError(ENOENT) - raw = data.encode() - file[:offset] + raw + file[offset+len(raw):] + raw = data + file = file[:offset] + raw + file[offset+len(raw):] + + self.fs.putFile(file, path[1:]) + print("Write successfull") return len(raw) - def open(self, path, flags): - print("[#] OPEN "+path) + def openFile(self, path): subTree = self.getSubtree(path) if subTree == False: - pass + raise FuseOSError(ENOENT) else: if not self.subtreeIsFile(subTree): # cannot open a dir @@ -364,6 +430,12 @@ class IotaFS_Fuse(LoggingMixIn, Operations): file, lastFetch, lastAccess = self.fs.getFile(sHash) return (file, path, sHash, lastFetch, lastAccess) + def open(self, path, flags): + return 0 + + def release(self, path, fh): + return 0 + if __name__ == '__main__': import argparse parser = argparse.ArgumentParser()