diff --git a/__pycache__/main.cpython-38.pyc b/__pycache__/main.cpython-38.pyc index 7fc8791..5e7cdc8 100644 Binary files a/__pycache__/main.cpython-38.pyc and b/__pycache__/main.cpython-38.pyc differ diff --git a/next.py b/next.py index 578944f..eb9b32c 100644 --- a/next.py +++ b/next.py @@ -1,5 +1,14 @@ #import pyfuse3 +# TODO: Milestones always written into new block +# TODO: Implement File COW (except for append) (+ version / token updates caused by this) +# TODO: ? Stop using tokens for dirs, use hashed name + parent token +# TODO: ? Stop using tokens for files, use hashed name + short ?version-number + parent token instead +# TODO: Store version-numbers for files in parent-dir +# TODO: Chane Milestone-Format from [stone,stone,...] to [[dirStne,dirStone,...],[fileStone,,...]] and dont save type for the stones +# TODO: Move iota-push-code from TangleBlob to chunk? and perform when sealing? +# TODO: When unmounting walk throught tree and seal all blobs + from iota import Iota, ProposedTransaction, Address, TryteString, Tag from iota.crypto.addresses import AddressGenerator from iota.crypto.types import Seed @@ -22,42 +31,78 @@ import copy import gzip import secrets -CHUNKSIZE = 2187 - 1 +CHUNKSIZE = 2187 def log(txt): print("[-] "+str(txt)) class Atom(): - def __init__(self) -> None: - self.type = "null" + def __init__(self, milestone: bool, cont, name: str = None) -> None: + self.milestone = milestone + self.delta = not milestone + self.name = name + self.cont = cont - def load(self, data) -> None: - if data[0]==True: - self.type = "milestone" - self.milestone = data[1:] + def dump(self): + if self.milestone: + return msgpack.dumps([True, self.cont]) else: - if data[1]==True: - self.type = "dir" - self.milestoneIndex = data[4] - else: - self.type = "file" - self.size = data[4] - self.name = data[2] - self.token = data[3] + return msgpack.dumps([False, self.name, self.cont]) - def dump(self) -> bytes: - # TODO: Delimiter? - # TODO: compression? - return msgpack.dumps(self._dumpAsArray()) +class Inode(): + def __init__(self, name: str, iotaApi: Iota, type: str = None) -> None: + self.name = name + self.type = type + self.ref = None + self.iotaApi = iotaApi - def _dumpAsArray(self): - if self.type=="milestone": - return [True]+self.milestone + def setType(self, type: str) -> None: + self.type = type + + def change(self, size: int=None, hash: int=None, milestoneIndex: int=None) -> Atom: + delta = {} + if size!=None: + self.size = size + delta[b's'] = size + if size!=None: + self.size = hash + delta[b'h'] = hash + if milestoneIndex!=None: + self.milestoneIndex = milestoneIndex + delta[b'm'] = milestoneIndex + return Atom(False, delta, self.name) + + def applyAtom(self, atom: Atom) -> None: + if atom.name != self.name: + raise Exception("Cannot apply atom ment for a different inode (names differ)") + if atom.milestone: + stones = atom.cont + if self.name in stones: + self.applyAtom(Atom(False, stones[self.name], self.name)) else: - if self.type=="file": - return [False, self.type, self.name, self.token, self.size] + delta = atom.cont + if b's' in delta: + self.size = delta[b's'] + if b'm' in delta: + self.milestoneIndex = delta[b'm'] + + def toStone(self) -> None: + if self.type=="file": + return [self.size, self.hash] + else: + return [self.milestoneIndex] + + def getRef(self): + if self.name == "*": + return None + if not self.ref: + if self.type=="dir": + self.elem = TangleFileTreeElement(self.name, self.milestoneIndex, self, self.iotaApi) + elif self.type=="file": + self.ref = TangleFile(self.name, self, self.api) else: - return [False, self.type, self.name, self.token, self.milestoneIndex] + raise Exception("Cannot get reference of an inode of type "+self.type) + return self.ref class BlobChunk(): def __init__(self, data: bytes = b'', sealed: bool = False) -> None: @@ -71,6 +116,8 @@ class BlobChunk(): if len(data)+len(self.data) > CHUNKSIZE: raise Exception("That to big!") self.data += data + if len(self.data) == CHUNKSIZE: + self.seal() def getBytesLeft(self) -> int: if self.sealed: @@ -164,13 +211,14 @@ class TangleBlob(): self.pushedNum = self.getChunkLen() return data - def fetch(self, skipChunks: int = 0) -> None: + def fetch(self) -> None: + skipChunks = self.preChunks chunkNum = self.getChunkLen() + skipChunks while True: key = self._getKey(chunkNum) cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16]) addr = self.adressGen.get_addresses(start=chunkNum, count=1)[0] - txHash = self.api.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"] + txHash = self.iotaApi.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"] if len(txHash)==0: break bundles = self.api.get_bundles(txHash[0])["bundles"] @@ -194,34 +242,48 @@ class TangleBlob(): def genToken(self) -> bytes: return secrets.token_bytes(32) + def sealLastChunk(self) -> None: + self.chunks[-1].seal() + class TangleFileTreeElement(TangleBlob): - def __init__(self, token: bytes, parent, iotaApi: Iota) -> None: - super(TangleFileTreeElement, self).__init__(token, iotaApi) + def __init__(self, name: str, lastMilestoneIndex: int, parent, iotaApi: Iota) -> None: + if isinstance(parent, bytes): + self.token = hashlib.sha3_384(parent + name.encode()).digest() + else: + self.token = hashlib.sha3_384(parent.getRef().token + name.encode()).digest() + super(TangleFileTreeElement, self).__init__(self.token, iotaApi) + self.name = name self.inodes = {} - self.atomStack = -1 - self.api = iotaApi self.parent = parent + self.milestoneIndex = lastMilestoneIndex + self.preChunks = self.milestoneIndex def _afterFetch(self) -> None: raw = self.read() if raw==b'': - self.atomStack = 0 return - data = msgpack.loads(raw, raw=True) - newAtoms = [] - for i, elem in enumerate(reversed(data)): - atom = Atom() - atom.load(elem) - if atom.type == "milestone": - self._applyMilestone(atom.milestone) + unpacker = msgpack.Unpacker(raw=True) + unpacker.feed(raw) + for i, elem in enumerate(reversed(unpacker)): + if elem[0]: + # Is a milestone + # TODO: Update our known milestoneIndex, if we find one + # might have to rewrite .fetch() and merge it here... + self.milestoneIndex = self.getChunkLen() + self._applyMilestone(elem[1]) break - newAtoms.append(atom) - self.atomStack = len(newAtoms) - for atom in reversed(newAtoms): - self._applyAtom(atom) + else: + if elem[1] in self.inodes: #name + self.inodes[elem[1]].applyDelta(elem[2]) + else: + # new file + self.inodes[elem[1]] = Inode(elem[1], self.iotaApi) + self.inodes[elem[1]].applyDelta(elem[2]) + + def _getSkipChunks(self): + return def _applyMilestone(self, milestone) -> None: - self.atomStack = 0 self.inodes = {} for stone in milestone: atom = Atom() @@ -229,15 +291,8 @@ class TangleFileTreeElement(TangleBlob): self.inodes[atom.name] = atom def _applyAtom(self, atom: Atom) -> None: - if atom.name in self.inodes: - self.atomStack += 1 - self.inodes[atom.name] = atom - if atom.type=="dir": - self.inodes[atom.name].elem = TangleFileTreeElement(atom.token, self, self.iotaApi) - elif atom.type=="file": - self.inodes[atom.name].elem = TangleFile(atom.name, self, self.api) - else: - raise Exception("How did such an atom get here?") + ## TODO: + pass def _newAtom(self, atom: Atom) -> None: self.append(atom.dump()) @@ -254,13 +309,10 @@ class TangleFileTreeElement(TangleBlob): self._requireFetched() if name in self.getNameList(): return False - atom = Atom() - atom.type="dir" - atom.name = name - atom.token = self.genToken() - atom.milestoneIndex = 0 + inode = Inode(name, self.iotaApi, "dir") + atom = inode.change(milestoneIndex=0) self._newAtom(atom) - self._applyAtom(atom) + self.inodes[name] = inode return True def mkfile(self, name: str) -> bool: @@ -276,28 +328,37 @@ class TangleFileTreeElement(TangleBlob): self._applyAtom(atom) return True - def _updateFileSize(self, name: str, size: int): + def _updateFileSize(self, name: str, size: int) -> None: self._requireFetched() self.inodes[name].size = size self._newAtom(self.inodes[name]) - def _updateFileToken(self, name: str, token: bytes, size: int): + def _updateFileToken(self, name: str, token: bytes, size: int) -> None: + log("New FileToken for file '"+name+"' registered") self._requireFetched() self.inodes[name].token = token self.inodes[name].size = size self._newAtom(self.inodes[name]) - def performMilestone(self): - stones = [] + def performMilestone(self) -> None: + stones = {} for a in self.inodes: - stones.append(self.inodes[a]._dumpAsArray()) + stones[a] = self.inodes[a].toStone() self.atomStack = 0 # TODO: Delimiter ? # TODO: compression ? - data = msgpack.dumps(stones) - self.append(data) + milestoneAtom = Atom(True, stones) + data = milestoneAtom.dump() + self.milestoneIndex = self.getSize() + if self.parent!=None: + self.parent._updateChildMilestone(self.name, self.milestoneIndex) + self.append(data, True) # inform parent about milestone (when merged) + def _updateChildMilestone(self, name: str, milestoneIndex: int): + self.inodes[name].milestoneIndex = milestoneIndex + self._newAtom(self.inodes[name]) + class TangleFile(): def __init__(self, name: str, parent: TangleFileTreeElement, iotaApi: Iota) -> None: self.api = iotaApi @@ -305,20 +366,27 @@ class TangleFile(): self.parent = parent self.reflexiveAtom = parent.inodes[name] self.size = self.reflexiveAtom.size - self.token = self.reflexiveAtom.token + self.hash = self.reflexiveAtom.hash + self.token = hashlib.sha3_384(b'f' + parent.getRef().token + self.hash).digest() self.blob = TangleBlob(self.token, iotaApi) def write(self, offset: int, data: bytes): if offset == self.size: self.append(data) else: - pass - + self.token = self.genToken() + oldData = self.blob.read() + newData = oldData[:offset] + data + oldData[offset+len(data):] + self.blob = TangleBlob(self.token) + self.blob.append(newData) + self.size = self.blob.getSize() + self.parent._updateFileToken(self.name, self.token, self.size) class IotaFs(): def __init__(self, token) -> None: self.api = Iota('https://nodes.thetangle.org:443', local_pow=True) - self.genesis = TangleFileTreeElement(token, None, self.api) + # TODO Cache last known milestone-Index of genesis locally + self.genesis = TangleFileTreeElement("*", 0, token, None, self.api) log("Fetching Genesis...") self.genesis.fetch() log("Retrieving reference to root") @@ -337,6 +405,9 @@ class IotaFs(): def createNewFile(self, name) -> None: pass -api = Iota('https://nodes.thetangle.org:443', local_pow=True) -token = b'testToken' -genesis = TangleFileTreeElement(token, None, api) +def main(): + api = Iota('https://nodes.thetangle.org:443', local_pow=True) + token = b'testToken' + genesis = TangleFileTreeElement("*", 0, token, api) + genesis.mkdir("/") + root = genesis.inodes["/"].getRef()