dodox/iotaFS
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

many changes and it doesnt work...

Browse Source
This commit is contained in:
Dominik Moritz Roth 2020-06-18 17:33:23 +02:00
parent c560414e36
commit 0e1826d282
2 changed files with 144 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
__pycache__/main.cpython-38.pyc
View File

Binary file not shown.

215
next.py
View File

@ -1,5 +1,14 @@
#import pyfuse3 #import pyfuse3
# TODO: Milestones always written into new block
# TODO: Implement File COW (except for append) (+ version / token updates caused by this)
# TODO: ? Stop using tokens for dirs, use hashed name + parent token
# TODO: ? Stop using tokens for files, use hashed name + short ?version-number + parent token instead
# TODO: Store version-numbers for files in parent-dir
# TODO: Chane Milestone-Format from [stone,stone,...] to [[dirStne,dirStone,...],[fileStone,,...]] and dont save type for the stones
# TODO: Move iota-push-code from TangleBlob to chunk? and perform when sealing?
# TODO: When unmounting walk throught tree and seal all blobs
from iota import Iota, ProposedTransaction, Address, TryteString, Tag from iota import Iota, ProposedTransaction, Address, TryteString, Tag
from iota.crypto.addresses import AddressGenerator from iota.crypto.addresses import AddressGenerator
from iota.crypto.types import Seed from iota.crypto.types import Seed
@ -22,42 +31,78 @@ import copy
import gzip import gzip
import secrets import secrets
CHUNKSIZE = 2187 - 1 CHUNKSIZE = 2187
def log(txt): def log(txt):
print("[-] "+str(txt)) print("[-] "+str(txt))
class Atom(): class Atom():
def __init__(self) -> None: def __init__(self, milestone: bool, cont, name: str = None) -> None:
self.type = "null" self.milestone = milestone
self.delta = not milestone
self.name = name
self.cont = cont
def load(self, data) -> None: def dump(self):
if data[0]==True: if self.milestone:
self.type = "milestone" return msgpack.dumps([True, self.cont])
self.milestone = data[1:]
else: else:
if data[1]==True: return msgpack.dumps([False, self.name, self.cont])
self.type = "dir"
self.milestoneIndex = data[4]
else:
self.type = "file"
self.size = data[4]
self.name = data[2]
self.token = data[3]
def dump(self) -> bytes: class Inode():
# TODO: Delimiter? def __init__(self, name: str, iotaApi: Iota, type: str = None) -> None:
# TODO: compression? self.name = name
return msgpack.dumps(self._dumpAsArray()) self.type = type
self.ref = None
self.iotaApi = iotaApi
def _dumpAsArray(self): def setType(self, type: str) -> None:
if self.type=="milestone": self.type = type
return [True]+self.milestone
def change(self, size: int=None, hash: int=None, milestoneIndex: int=None) -> Atom:
delta = {}
if size!=None:
self.size = size
delta[b's'] = size
if size!=None:
self.size = hash
delta[b'h'] = hash
if milestoneIndex!=None:
self.milestoneIndex = milestoneIndex
delta[b'm'] = milestoneIndex
return Atom(False, delta, self.name)
def applyAtom(self, atom: Atom) -> None:
if atom.name != self.name:
raise Exception("Cannot apply atom ment for a different inode (names differ)")
if atom.milestone:
stones = atom.cont
if self.name in stones:
self.applyAtom(Atom(False, stones[self.name], self.name))
else: else:
delta = atom.cont
if b's' in delta:
self.size = delta[b's']
if b'm' in delta:
self.milestoneIndex = delta[b'm']
def toStone(self) -> None:
if self.type=="file": if self.type=="file":
return [False, self.type, self.name, self.token, self.size] return [self.size, self.hash]
else: else:
return [False, self.type, self.name, self.token, self.milestoneIndex] return [self.milestoneIndex]
def getRef(self):
if self.name == "*":
return None
if not self.ref:
if self.type=="dir":
self.elem = TangleFileTreeElement(self.name, self.milestoneIndex, self, self.iotaApi)
elif self.type=="file":
self.ref = TangleFile(self.name, self, self.api)
else:
raise Exception("Cannot get reference of an inode of type "+self.type)
return self.ref
class BlobChunk(): class BlobChunk():
def __init__(self, data: bytes = b'', sealed: bool = False) -> None: def __init__(self, data: bytes = b'', sealed: bool = False) -> None:
@ -71,6 +116,8 @@ class BlobChunk():
if len(data)+len(self.data) > CHUNKSIZE: if len(data)+len(self.data) > CHUNKSIZE:
raise Exception("That to big!") raise Exception("That to big!")
self.data += data self.data += data
if len(self.data) == CHUNKSIZE:
self.seal()
def getBytesLeft(self) -> int: def getBytesLeft(self) -> int:
if self.sealed: if self.sealed:
@ -164,13 +211,14 @@ class TangleBlob():
self.pushedNum = self.getChunkLen() self.pushedNum = self.getChunkLen()
return data return data
def fetch(self, skipChunks: int = 0) -> None: def fetch(self) -> None:
skipChunks = self.preChunks
chunkNum = self.getChunkLen() + skipChunks chunkNum = self.getChunkLen() + skipChunks
while True: while True:
key = self._getKey(chunkNum) key = self._getKey(chunkNum)
cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16]) cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16])
addr = self.adressGen.get_addresses(start=chunkNum, count=1)[0] addr = self.adressGen.get_addresses(start=chunkNum, count=1)[0]
txHash = self.api.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"] txHash = self.iotaApi.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"]
if len(txHash)==0: if len(txHash)==0:
break break
bundles = self.api.get_bundles(txHash[0])["bundles"] bundles = self.api.get_bundles(txHash[0])["bundles"]
@ -194,34 +242,48 @@ class TangleBlob():
def genToken(self) -> bytes: def genToken(self) -> bytes:
return secrets.token_bytes(32) return secrets.token_bytes(32)
def sealLastChunk(self) -> None:
self.chunks[-1].seal()
class TangleFileTreeElement(TangleBlob): class TangleFileTreeElement(TangleBlob):
def __init__(self, token: bytes, parent, iotaApi: Iota) -> None: def __init__(self, name: str, lastMilestoneIndex: int, parent, iotaApi: Iota) -> None:
super(TangleFileTreeElement, self).__init__(token, iotaApi) if isinstance(parent, bytes):
self.token = hashlib.sha3_384(parent + name.encode()).digest()
else:
self.token = hashlib.sha3_384(parent.getRef().token + name.encode()).digest()
super(TangleFileTreeElement, self).__init__(self.token, iotaApi)
self.name = name
self.inodes = {} self.inodes = {}
self.atomStack = -1
self.api = iotaApi
self.parent = parent self.parent = parent
self.milestoneIndex = lastMilestoneIndex
self.preChunks = self.milestoneIndex
def _afterFetch(self) -> None: def _afterFetch(self) -> None:
raw = self.read() raw = self.read()
if raw==b'': if raw==b'':
self.atomStack = 0
return return
data = msgpack.loads(raw, raw=True) unpacker = msgpack.Unpacker(raw=True)
newAtoms = [] unpacker.feed(raw)
for i, elem in enumerate(reversed(data)): for i, elem in enumerate(reversed(unpacker)):
atom = Atom() if elem[0]:
atom.load(elem) # Is a milestone
if atom.type == "milestone": # TODO: Update our known milestoneIndex, if we find one
self._applyMilestone(atom.milestone) # might have to rewrite .fetch() and merge it here...
self.milestoneIndex = self.getChunkLen()
self._applyMilestone(elem[1])
break break
newAtoms.append(atom) else:
self.atomStack = len(newAtoms) if elem[1] in self.inodes: #name
for atom in reversed(newAtoms): self.inodes[elem[1]].applyDelta(elem[2])
self._applyAtom(atom) else:
# new file
self.inodes[elem[1]] = Inode(elem[1], self.iotaApi)
self.inodes[elem[1]].applyDelta(elem[2])
def _getSkipChunks(self):
return
def _applyMilestone(self, milestone) -> None: def _applyMilestone(self, milestone) -> None:
self.atomStack = 0
self.inodes = {} self.inodes = {}
for stone in milestone: for stone in milestone:
atom = Atom() atom = Atom()
@ -229,15 +291,8 @@ class TangleFileTreeElement(TangleBlob):
self.inodes[atom.name] = atom self.inodes[atom.name] = atom
def _applyAtom(self, atom: Atom) -> None: def _applyAtom(self, atom: Atom) -> None:
if atom.name in self.inodes: ## TODO:
self.atomStack += 1 pass
self.inodes[atom.name] = atom
if atom.type=="dir":
self.inodes[atom.name].elem = TangleFileTreeElement(atom.token, self, self.iotaApi)
elif atom.type=="file":
self.inodes[atom.name].elem = TangleFile(atom.name, self, self.api)
else:
raise Exception("How did such an atom get here?")
def _newAtom(self, atom: Atom) -> None: def _newAtom(self, atom: Atom) -> None:
self.append(atom.dump()) self.append(atom.dump())
@ -254,13 +309,10 @@ class TangleFileTreeElement(TangleBlob):
self._requireFetched() self._requireFetched()
if name in self.getNameList(): if name in self.getNameList():
return False return False
atom = Atom() inode = Inode(name, self.iotaApi, "dir")
atom.type="dir" atom = inode.change(milestoneIndex=0)
atom.name = name
atom.token = self.genToken()
atom.milestoneIndex = 0
self._newAtom(atom) self._newAtom(atom)
self._applyAtom(atom) self.inodes[name] = inode
return True return True
def mkfile(self, name: str) -> bool: def mkfile(self, name: str) -> bool:
@ -276,28 +328,37 @@ class TangleFileTreeElement(TangleBlob):
self._applyAtom(atom) self._applyAtom(atom)
return True return True
def _updateFileSize(self, name: str, size: int): def _updateFileSize(self, name: str, size: int) -> None:
self._requireFetched() self._requireFetched()
self.inodes[name].size = size self.inodes[name].size = size
self._newAtom(self.inodes[name]) self._newAtom(self.inodes[name])
def _updateFileToken(self, name: str, token: bytes, size: int): def _updateFileToken(self, name: str, token: bytes, size: int) -> None:
log("New FileToken for file '"+name+"' registered")
self._requireFetched() self._requireFetched()
self.inodes[name].token = token self.inodes[name].token = token
self.inodes[name].size = size self.inodes[name].size = size
self._newAtom(self.inodes[name]) self._newAtom(self.inodes[name])
def performMilestone(self): def performMilestone(self) -> None:
stones = [] stones = {}
for a in self.inodes: for a in self.inodes:
stones.append(self.inodes[a]._dumpAsArray()) stones[a] = self.inodes[a].toStone()
self.atomStack = 0 self.atomStack = 0
# TODO: Delimiter ? # TODO: Delimiter ?
# TODO: compression ? # TODO: compression ?
data = msgpack.dumps(stones) milestoneAtom = Atom(True, stones)
self.append(data) data = milestoneAtom.dump()
self.milestoneIndex = self.getSize()
if self.parent!=None:
self.parent._updateChildMilestone(self.name, self.milestoneIndex)
self.append(data, True)
# inform parent about milestone (when merged) # inform parent about milestone (when merged)
def _updateChildMilestone(self, name: str, milestoneIndex: int):
self.inodes[name].milestoneIndex = milestoneIndex
self._newAtom(self.inodes[name])
class TangleFile(): class TangleFile():
def __init__(self, name: str, parent: TangleFileTreeElement, iotaApi: Iota) -> None: def __init__(self, name: str, parent: TangleFileTreeElement, iotaApi: Iota) -> None:
self.api = iotaApi self.api = iotaApi
@ -305,20 +366,27 @@ class TangleFile():
self.parent = parent self.parent = parent
self.reflexiveAtom = parent.inodes[name] self.reflexiveAtom = parent.inodes[name]
self.size = self.reflexiveAtom.size self.size = self.reflexiveAtom.size
self.token = self.reflexiveAtom.token self.hash = self.reflexiveAtom.hash
self.token = hashlib.sha3_384(b'f' + parent.getRef().token + self.hash).digest()
self.blob = TangleBlob(self.token, iotaApi) self.blob = TangleBlob(self.token, iotaApi)
def write(self, offset: int, data: bytes): def write(self, offset: int, data: bytes):
if offset == self.size: if offset == self.size:
self.append(data) self.append(data)
else: else:
pass self.token = self.genToken()
oldData = self.blob.read()
newData = oldData[:offset] + data + oldData[offset+len(data):]
self.blob = TangleBlob(self.token)
self.blob.append(newData)
self.size = self.blob.getSize()
self.parent._updateFileToken(self.name, self.token, self.size)
class IotaFs(): class IotaFs():
def __init__(self, token) -> None: def __init__(self, token) -> None:
self.api = Iota('https://nodes.thetangle.org:443', local_pow=True) self.api = Iota('https://nodes.thetangle.org:443', local_pow=True)
self.genesis = TangleFileTreeElement(token, None, self.api) # TODO Cache last known milestone-Index of genesis locally
self.genesis = TangleFileTreeElement("*", 0, token, None, self.api)
log("Fetching Genesis...") log("Fetching Genesis...")
self.genesis.fetch() self.genesis.fetch()
log("Retrieving reference to root") log("Retrieving reference to root")
@ -337,6 +405,9 @@ class IotaFs():
def createNewFile(self, name) -> None: def createNewFile(self, name) -> None:
pass pass
api = Iota('https://nodes.thetangle.org:443', local_pow=True) def main():
token = b'testToken' api = Iota('https://nodes.thetangle.org:443', local_pow=True)
genesis = TangleFileTreeElement(token, None, api) token = b'testToken'
genesis = TangleFileTreeElement("*", 0, token, api)
genesis.mkdir("/")
root = genesis.inodes["/"].getRef()