From 0e1826d2828787b6e18615a2d9f7befa7ae6fd14 Mon Sep 17 00:00:00 2001 From: Dominik Roth Date: Thu, 18 Jun 2020 17:33:23 +0200 Subject: [PATCH] many changes and it doesnt work... --- __pycache__/main.cpython-38.pyc | Bin 13314 -> 13392 bytes next.py | 217 +++++++++++++++++++++----------- 2 files changed, 144 insertions(+), 73 deletions(-) diff --git a/__pycache__/main.cpython-38.pyc b/__pycache__/main.cpython-38.pyc index 7fc8791792ca66f422ed570644748f48b12cfed5..5e7cdc873ba1f8a12fcc2950ddd6b3a5ce8760a3 100644 GIT binary patch delta 2467 zcmZWqeQaCR6@T|VKl|D7SL~#5oV0f1B-BltG=$O$y08iejH(~BOxm)kk?Z@CII;80 zeJ?FZ-O@D`tg28U2eEGbh(o)zr5_nGpbhaCDjyqb2a~oTBLw`3goc>J{$SFybFM=g zfuH=l=iYPfx##@ux%VC&eSb9g<3PZt;P0gJ>E4U?y&Qa1V;z+n+OJjCRoUx)jIoC* z70(LBc31vvTw?5JVEYzWwvws~YitDW3ck*cz?Z=VHdC1kbue}u-U?r1?O@kGT)&d5 zTsy`FHNJ|wxPFYmbX)4-AP*f=2E9DY>#_Cm2#;c0#~XMfwtl{XH(?v#G2V=AkjHro zwjtiSM@c8D?jKoBW?;`iR)dqN7@LJ%k-IYwI0n~e*ilx}xO|Ad%vQg3`0j%UgV%X!s5|eh*Xor;55-c$! zOBwUcGjR^(x|+=0&B`Gi&Nglgp=Fao)=s1xE$Nld8h5iNpP+bkyO_^8V!w>t$rVf- z$z$;10FDSsNvtECr(lm1GuiQ~p0_gG7L%k2Af(-*i464YXeKuy9+dLFypKY(u<0$8Mbn|HUU zk~hED=LE{ZMc$OLP&r(3!8^@~ZAWlUIwG?XPope;NkE=z0Ut-Hs`JT0(H4}xunD#j zbP-UhLJ-VAZ+s=wVP`z)t$K=?DM}lxcp~-*s_hjB3husN4XY81Im}#eAwI|c1rM|w z)Q_U_E4a}z6DMiWAxXywWMET}X}z0$4X0Y)j?01D>rauN4Rm?b}a$Ce%_;S@o)ytqW@yz;pvStXE z)nzmF+RXE$yg=|XTM0gX9zAKEEa)utg7yO#@pbYgSAPnU?eRL9_-(%YKHRrGllvTDnwx;PU)i)o z+CW+zG=`Ce$51EPPcI{W&_k-7`^J`$4?pb)vio7QBkMj%%l-mB=;+oLP}~NGI}?!V z+{`LOGFY8KwgbnWjmY>Y>&x2Wu0ryTBzv^7^^Q1WLHPNa7H?>aQqu8;48xH%6YHy9 z+ZmE)*G`9OC*t?C+N%U%f*^q`xdlk3x5g;vsyaL>L#=8P)`6;OJGkFd=`Hxyyq4a` zH25N2O37d(`>&CU*b>OgfE0(}#M&D@(xJ=FO`BD>H9Tyaj(C@BWi9Je%Xij_`8^u_ z3moohVl(hU*DG!r=>qIpmx#)pkke>8sw`itV6R7*8ICo40?w@)Ymy;dMT7JrKUc`* zc4EzIH4`0bQPkH`af|Zo#SU)Q-QB@-|&- zsk^ip>AGb{tcO$|5G*Ar{zhH8M#W-^-kw%A2c10^_P&Cnc+6y`xBXKU)clu8?^)(w zR-^Pq{4F{p={x*W#MfZWhDRfmtJHo=%6T}kVHNu$ytLuIrtd4s`dy-dxXqgx+k69B zdspM}>FC$%lpvaa!cnBy0f=CfxC}PxVDBGQZZrOJ0oo%6Y z2?|xwpurrJheo#e0udWr)I|SaRE(&x#z$oQ;U5DUH5y~0i6(l^-2w)8v)`P1&$)BI z^PO|=j(2bG4&3hd`xN>tIQ#9o#Ny+D*EOigzo+e2VOoBju@+!yKJP6DxFLVqJPfcF z9p4z-lW!;vYOo$J54;6K_)B06viXChjR4#6{opaEMrYD>SJqgnrHw#`W<@RCGRX2+ zUdtR&I&`bZ^3l_dpr>h-TEP(o|ERxw#%)Th)-_h8Gx7G@)+RLOIPvk& zlYh(U=RAtHgy!G1D63ie0%fa~ReHQRGppr_MwL+jWnv8H$4Q#$^jQXBvE?N!u}t=n z<(IxdR>_r)Dxg>jF(|@wp;lZNZiFDN4==@|;e+Lm@wPPt3Iw1;HNXcRFtDqv9@=rF zY)SbLxm-^o-9O}-mTmX5B{KMR*-XfxXL1|99reR+xNLIkMx{p=3o}N}o7F~v*lU)y zNfkF{6w9-N*x%bl{8i3J<^oxba)f&WS&(sq6sL0>bInqY6zv-Yae(aNf^8X<%2p4$ zljoJv8!Hr!&V+K|tS4Vnz6Q2!X0E!^pGs!LfaIM?_S!U)BI06@W(21rn%U>=bvE|L z6PsK;m5y7E$gsyx5Yt5k2k43Jc(PBdkakxpo!HdjO$>-N$-a$Z3I|K5WQ*VWf)XMr z=aZ#?35IHtl)k(<<$BPcpt<+)j>^UObAunUc&Ktsh0;?Z)@A&;KvpHOR7$Wk7tE?y z8i}@Sqxe`zrXU_7TRhCbZz-_cNtUZy1HJu@*o=2as;N_l2tBwrQi~@e(eke-!YLzA zz@V}Q)ex}>;vbQN@Ez`*dY}FPxz1rl)s6^TLNIJ+*ufw--GV2o=EINpW7VmMoS4Zj z)=q3;c!r@9$D7OW+iEZToE3BkF%v7Ahk~*?EZJIfm!)*OoG)caxPAlkle6!>i@u9J_Wg zyhso;3Y%WKl>JQPl8U-MVfUs7?e=8Kj^MR*>8isV^a8)hQ)mw@owCGDay@=pS6or5<(m+p8STbt1Wf!V}`|n^prY_~x{u3ohu|9#cEa4HUnYR?7NJ92&W> z!^^JKT|MW^v%j$puEe(+6UO7b`TaO;daHhn%w_o5^e7HazXFcq*z`KMmd3m1N`|b$ z5$$5pTu*P>vM()@kq~w~W3SBbxhw)uieodT76lh6S)H!CN%;AUK0T?Zih|2x;mE7J zcb*~05MYq)c?|nv%cgK=xax*(N!r!=(wkk?$%q5^L2MD+ik_xvKZt z%%+29n%--Zukbj@VcRv*8#Xw0Mx5eZxWQbVJAF=})j#Cf({i;<<`(>J<_SacJBH6T zN5lN0!cwk*3n}XH@#d*;KmOF*6Osg8CP!>SpCj1a5`|s3vt@PV7d)>Ps?7aEHs5Aq zlnzNH>_%VfH}El@ZEe{02~S^?qKStWq>mL}lwP#4>r;jc>4`A#Ti#oBl6q3_9 zgoCqg4Y350#(8~4@yx6mcoqMgb#=x6og96Nw>3haWM>9^m?k&l$dC;EPx0ETp^^56|!kUUHPo7z1 zm>&8D=&1>gcFoStUiu5_>=eVee9jMLvRmcl?~OY{eTigR+=ol&HbqZTw0M_6UcdX; z3N!GvA&%lBb8moQ{C;*-v1=}(b2N~$muUF++y&J&p>0Zuk4hr{uBa?3SsqaN``5oU Cz(Ifj diff --git a/next.py b/next.py index 578944f..eb9b32c 100644 --- a/next.py +++ b/next.py @@ -1,5 +1,14 @@ #import pyfuse3 +# TODO: Milestones always written into new block +# TODO: Implement File COW (except for append) (+ version / token updates caused by this) +# TODO: ? Stop using tokens for dirs, use hashed name + parent token +# TODO: ? Stop using tokens for files, use hashed name + short ?version-number + parent token instead +# TODO: Store version-numbers for files in parent-dir +# TODO: Chane Milestone-Format from [stone,stone,...] to [[dirStne,dirStone,...],[fileStone,,...]] and dont save type for the stones +# TODO: Move iota-push-code from TangleBlob to chunk? and perform when sealing? +# TODO: When unmounting walk throught tree and seal all blobs + from iota import Iota, ProposedTransaction, Address, TryteString, Tag from iota.crypto.addresses import AddressGenerator from iota.crypto.types import Seed @@ -22,42 +31,78 @@ import copy import gzip import secrets -CHUNKSIZE = 2187 - 1 +CHUNKSIZE = 2187 def log(txt): print("[-] "+str(txt)) class Atom(): - def __init__(self) -> None: - self.type = "null" + def __init__(self, milestone: bool, cont, name: str = None) -> None: + self.milestone = milestone + self.delta = not milestone + self.name = name + self.cont = cont - def load(self, data) -> None: - if data[0]==True: - self.type = "milestone" - self.milestone = data[1:] + def dump(self): + if self.milestone: + return msgpack.dumps([True, self.cont]) else: - if data[1]==True: - self.type = "dir" - self.milestoneIndex = data[4] - else: - self.type = "file" - self.size = data[4] - self.name = data[2] - self.token = data[3] + return msgpack.dumps([False, self.name, self.cont]) - def dump(self) -> bytes: - # TODO: Delimiter? - # TODO: compression? - return msgpack.dumps(self._dumpAsArray()) +class Inode(): + def __init__(self, name: str, iotaApi: Iota, type: str = None) -> None: + self.name = name + self.type = type + self.ref = None + self.iotaApi = iotaApi - def _dumpAsArray(self): - if self.type=="milestone": - return [True]+self.milestone + def setType(self, type: str) -> None: + self.type = type + + def change(self, size: int=None, hash: int=None, milestoneIndex: int=None) -> Atom: + delta = {} + if size!=None: + self.size = size + delta[b's'] = size + if size!=None: + self.size = hash + delta[b'h'] = hash + if milestoneIndex!=None: + self.milestoneIndex = milestoneIndex + delta[b'm'] = milestoneIndex + return Atom(False, delta, self.name) + + def applyAtom(self, atom: Atom) -> None: + if atom.name != self.name: + raise Exception("Cannot apply atom ment for a different inode (names differ)") + if atom.milestone: + stones = atom.cont + if self.name in stones: + self.applyAtom(Atom(False, stones[self.name], self.name)) else: - if self.type=="file": - return [False, self.type, self.name, self.token, self.size] + delta = atom.cont + if b's' in delta: + self.size = delta[b's'] + if b'm' in delta: + self.milestoneIndex = delta[b'm'] + + def toStone(self) -> None: + if self.type=="file": + return [self.size, self.hash] + else: + return [self.milestoneIndex] + + def getRef(self): + if self.name == "*": + return None + if not self.ref: + if self.type=="dir": + self.elem = TangleFileTreeElement(self.name, self.milestoneIndex, self, self.iotaApi) + elif self.type=="file": + self.ref = TangleFile(self.name, self, self.api) else: - return [False, self.type, self.name, self.token, self.milestoneIndex] + raise Exception("Cannot get reference of an inode of type "+self.type) + return self.ref class BlobChunk(): def __init__(self, data: bytes = b'', sealed: bool = False) -> None: @@ -71,6 +116,8 @@ class BlobChunk(): if len(data)+len(self.data) > CHUNKSIZE: raise Exception("That to big!") self.data += data + if len(self.data) == CHUNKSIZE: + self.seal() def getBytesLeft(self) -> int: if self.sealed: @@ -164,13 +211,14 @@ class TangleBlob(): self.pushedNum = self.getChunkLen() return data - def fetch(self, skipChunks: int = 0) -> None: + def fetch(self) -> None: + skipChunks = self.preChunks chunkNum = self.getChunkLen() + skipChunks while True: key = self._getKey(chunkNum) cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16]) addr = self.adressGen.get_addresses(start=chunkNum, count=1)[0] - txHash = self.api.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"] + txHash = self.iotaApi.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"] if len(txHash)==0: break bundles = self.api.get_bundles(txHash[0])["bundles"] @@ -194,34 +242,48 @@ class TangleBlob(): def genToken(self) -> bytes: return secrets.token_bytes(32) + def sealLastChunk(self) -> None: + self.chunks[-1].seal() + class TangleFileTreeElement(TangleBlob): - def __init__(self, token: bytes, parent, iotaApi: Iota) -> None: - super(TangleFileTreeElement, self).__init__(token, iotaApi) + def __init__(self, name: str, lastMilestoneIndex: int, parent, iotaApi: Iota) -> None: + if isinstance(parent, bytes): + self.token = hashlib.sha3_384(parent + name.encode()).digest() + else: + self.token = hashlib.sha3_384(parent.getRef().token + name.encode()).digest() + super(TangleFileTreeElement, self).__init__(self.token, iotaApi) + self.name = name self.inodes = {} - self.atomStack = -1 - self.api = iotaApi self.parent = parent + self.milestoneIndex = lastMilestoneIndex + self.preChunks = self.milestoneIndex def _afterFetch(self) -> None: raw = self.read() if raw==b'': - self.atomStack = 0 return - data = msgpack.loads(raw, raw=True) - newAtoms = [] - for i, elem in enumerate(reversed(data)): - atom = Atom() - atom.load(elem) - if atom.type == "milestone": - self._applyMilestone(atom.milestone) + unpacker = msgpack.Unpacker(raw=True) + unpacker.feed(raw) + for i, elem in enumerate(reversed(unpacker)): + if elem[0]: + # Is a milestone + # TODO: Update our known milestoneIndex, if we find one + # might have to rewrite .fetch() and merge it here... + self.milestoneIndex = self.getChunkLen() + self._applyMilestone(elem[1]) break - newAtoms.append(atom) - self.atomStack = len(newAtoms) - for atom in reversed(newAtoms): - self._applyAtom(atom) + else: + if elem[1] in self.inodes: #name + self.inodes[elem[1]].applyDelta(elem[2]) + else: + # new file + self.inodes[elem[1]] = Inode(elem[1], self.iotaApi) + self.inodes[elem[1]].applyDelta(elem[2]) + + def _getSkipChunks(self): + return def _applyMilestone(self, milestone) -> None: - self.atomStack = 0 self.inodes = {} for stone in milestone: atom = Atom() @@ -229,15 +291,8 @@ class TangleFileTreeElement(TangleBlob): self.inodes[atom.name] = atom def _applyAtom(self, atom: Atom) -> None: - if atom.name in self.inodes: - self.atomStack += 1 - self.inodes[atom.name] = atom - if atom.type=="dir": - self.inodes[atom.name].elem = TangleFileTreeElement(atom.token, self, self.iotaApi) - elif atom.type=="file": - self.inodes[atom.name].elem = TangleFile(atom.name, self, self.api) - else: - raise Exception("How did such an atom get here?") + ## TODO: + pass def _newAtom(self, atom: Atom) -> None: self.append(atom.dump()) @@ -254,13 +309,10 @@ class TangleFileTreeElement(TangleBlob): self._requireFetched() if name in self.getNameList(): return False - atom = Atom() - atom.type="dir" - atom.name = name - atom.token = self.genToken() - atom.milestoneIndex = 0 + inode = Inode(name, self.iotaApi, "dir") + atom = inode.change(milestoneIndex=0) self._newAtom(atom) - self._applyAtom(atom) + self.inodes[name] = inode return True def mkfile(self, name: str) -> bool: @@ -276,28 +328,37 @@ class TangleFileTreeElement(TangleBlob): self._applyAtom(atom) return True - def _updateFileSize(self, name: str, size: int): + def _updateFileSize(self, name: str, size: int) -> None: self._requireFetched() self.inodes[name].size = size self._newAtom(self.inodes[name]) - def _updateFileToken(self, name: str, token: bytes, size: int): + def _updateFileToken(self, name: str, token: bytes, size: int) -> None: + log("New FileToken for file '"+name+"' registered") self._requireFetched() self.inodes[name].token = token self.inodes[name].size = size self._newAtom(self.inodes[name]) - def performMilestone(self): - stones = [] + def performMilestone(self) -> None: + stones = {} for a in self.inodes: - stones.append(self.inodes[a]._dumpAsArray()) + stones[a] = self.inodes[a].toStone() self.atomStack = 0 # TODO: Delimiter ? # TODO: compression ? - data = msgpack.dumps(stones) - self.append(data) + milestoneAtom = Atom(True, stones) + data = milestoneAtom.dump() + self.milestoneIndex = self.getSize() + if self.parent!=None: + self.parent._updateChildMilestone(self.name, self.milestoneIndex) + self.append(data, True) # inform parent about milestone (when merged) + def _updateChildMilestone(self, name: str, milestoneIndex: int): + self.inodes[name].milestoneIndex = milestoneIndex + self._newAtom(self.inodes[name]) + class TangleFile(): def __init__(self, name: str, parent: TangleFileTreeElement, iotaApi: Iota) -> None: self.api = iotaApi @@ -305,20 +366,27 @@ class TangleFile(): self.parent = parent self.reflexiveAtom = parent.inodes[name] self.size = self.reflexiveAtom.size - self.token = self.reflexiveAtom.token + self.hash = self.reflexiveAtom.hash + self.token = hashlib.sha3_384(b'f' + parent.getRef().token + self.hash).digest() self.blob = TangleBlob(self.token, iotaApi) def write(self, offset: int, data: bytes): if offset == self.size: self.append(data) else: - pass - + self.token = self.genToken() + oldData = self.blob.read() + newData = oldData[:offset] + data + oldData[offset+len(data):] + self.blob = TangleBlob(self.token) + self.blob.append(newData) + self.size = self.blob.getSize() + self.parent._updateFileToken(self.name, self.token, self.size) class IotaFs(): def __init__(self, token) -> None: self.api = Iota('https://nodes.thetangle.org:443', local_pow=True) - self.genesis = TangleFileTreeElement(token, None, self.api) + # TODO Cache last known milestone-Index of genesis locally + self.genesis = TangleFileTreeElement("*", 0, token, None, self.api) log("Fetching Genesis...") self.genesis.fetch() log("Retrieving reference to root") @@ -337,6 +405,9 @@ class IotaFs(): def createNewFile(self, name) -> None: pass -api = Iota('https://nodes.thetangle.org:443', local_pow=True) -token = b'testToken' -genesis = TangleFileTreeElement(token, None, api) +def main(): + api = Iota('https://nodes.thetangle.org:443', local_pow=True) + token = b'testToken' + genesis = TangleFileTreeElement("*", 0, token, api) + genesis.mkdir("/") + root = genesis.inodes["/"].getRef()