dodox/iotaFS
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

everything works! (except for compression)

Browse Source
This commit is contained in:
Dominik Moritz Roth 2020-06-09 21:50:18 +02:00
parent 07dd1fb4e2
commit d0ac89b622
2 changed files with 149 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
__pycache__/main.cpython-38.pyc Normal file
View File

Binary file not shown.

226
main.py
View File

@ -23,6 +23,8 @@ from errno import ENOENT
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
import stat import stat
import os import os
import gzip
import secrets
class IotaFS_BlobStore(): class IotaFS_BlobStore():
def __init__(self, api=None): def __init__(self, api=None):
@ -31,7 +33,7 @@ class IotaFS_BlobStore():
else: else:
self.api = api self.api = api
def genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1): def _genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1):
msg = TryteString.from_bytes(data) msg = TryteString.from_bytes(data)
bundles = [] bundles = []
nextAddr = addrIter.__next__() nextAddr = addrIter.__next__()
@ -59,7 +61,7 @@ class IotaFS_BlobStore():
) )
return bundles return bundles
def sendBundles(self, bundles): def _sendBundles(self, bundles):
bundleRets = [] bundleRets = []
for i,bundle in enumerate(bundles): for i,bundle in enumerate(bundles):
print(str(int(i/len(bundles)*100))+"%") print(str(int(i/len(bundles)*100))+"%")
@ -84,8 +86,8 @@ class IotaFS_BlobStore():
cipher = AES.new(sHash[:16], AES.MODE_CBC, sHash[22:][:16]) cipher = AES.new(sHash[:16], AES.MODE_CBC, sHash[22:][:16])
ct_bytes = cipher.encrypt(pad(data, AES.block_size)) ct_bytes = cipher.encrypt(pad(data, AES.block_size))
addrIter = AddressGenerator(Seed(trSeed)).create_iterator(start = 0, step = 1) addrIter = AddressGenerator(Seed(trSeed)).create_iterator(start = 0, step = 1)
bundles = self.genBundles(ct_bytes, addrIter) bundles = self._genBundles(ct_bytes, addrIter)
self.sendBundles(bundles) self._sendBundles(bundles)
def uploadTxt(self, txt, secret): def uploadTxt(self, txt, secret):
data = str.encode(txt) data = str.encode(txt)
@ -107,7 +109,7 @@ class IotaFS_BlobStore():
for tx in bundle.transactions: for tx in bundle.transactions:
tryteMsg+=str(tx.signature_message_fragment) tryteMsg+=str(tx.signature_message_fragment)
if tryteMsg == "": if tryteMsg == "":
return "" return b''
tryteStr = TryteString(tryteMsg.rstrip("9")) tryteStr = TryteString(tryteMsg.rstrip("9"))
try: try:
ct_bytes = tryteStr.as_bytes() ct_bytes = tryteStr.as_bytes()
@ -136,16 +138,22 @@ class IotaFS_BlobStore():
f.write(y) f.write(y)
class IotaFS(): class IotaFS():
def __init__(self, token): def __init__(self, token, fileCompression=False):
self.api = Iota('https://nodes.thetangle.org:443', local_pow=True) self.api = Iota('https://nodes.thetangle.org:443', local_pow=True)
self.blobStore = IotaFS_BlobStore(self.api) self.blobStore = IotaFS_BlobStore(self.api)
#self.token = token #self.token = token
self.fileCompression = fileCompression
self.hashState = hashlib.sha3_384() self.hashState = hashlib.sha3_384()
genesis = "This is the genesis block. lol."
if self.fileCompression:
raise Exception("Compression does not work currently")
genesis += "#FILE COMPRESSION#"
self.hashState.update(genesis.encode())
self.hashState.update(token.encode()) self.hashState.update(token.encode())
self._fileTree = {} self._fileTree = {}
self.lastBlockIncomplete = False self.lastBlockIncomplete = False
self.incompleteBlockRescanTimeout = 5 self.incompleteBlockRescanTimeout = 5
self.chainDelimiter = "#CHAIN_DELIM#" self.chainDelimiter = "#IOTA_FS#".encode()
self.cache = {} self.cache = {}
self._fetchFileTree() self._fetchFileTree()
@ -155,34 +163,34 @@ class IotaFS():
self._fetchFileTree() self._fetchFileTree()
return copy.deepcopy(self._fileTree) return copy.deepcopy(self._fileTree)
async def fileTreeFetchLoop(self, interval=10):
while True:
self._fetchFileTree()
await asyncio.sleep(10)
def _fetchFileTree(self): def _fetchFileTree(self):
print("[<] Fetching FileTree") print("[<] Fetching FileTree")
chain = "" chain = bytes()
while True: while True:
print("[<] Fetching FileTree-ChainBlock") print("[<] Fetching FileTree-ChainBlock")
sHash = self.hashState.digest() sHash = self.hashState.digest()
block = self._getBlob(sHash) block = self._getBlob(sHash)
if block=="": data = block
if self.fileCompression:
data = gzip.decompress(data)
if data==b'':
print("[-] Last Block Received") print("[-] Last Block Received")
break break
self.hashState.update(block) self.hashState.update(block)
chain+=block chain+=data
if chain=="": if chain==b'':
print("[.] FileTree succesfully fetched: [EMPTY FILE TREE]") print("[.] FileTree succesfully fetched: [NO UPDATES]")
return return
if chain.endswith(self.chainDelimiter): if chain.endswith(self.chainDelimiter):
curRing = chain.split(self.chainDelimiter)[-1] curRing = chain.split(self.chainDelimiter)[-2]
self.lastBlockIncomplete = False self.lastBlockIncomplete = False
else: else:
print("[-] Last Block was incomplete; refetching...") print("[-] Last Block was incomplete; refetching...")
self.lastBlockIncomplete = True self.lastBlockIncomplete = True
time.sleep(self.incompleteBlockRescanTimeout) time.sleep(self.incompleteBlockRescanTimeout)
self._fetchFileTree() self._fetchFileTree()
return
print("{RING}: "+str(curRing))
self._fileTree = msgpack.loads(curRing) self._fileTree = msgpack.loads(curRing)
print("[.] FileTree succesfully fetched: ") print("[.] FileTree succesfully fetched: ")
@ -200,7 +208,7 @@ class IotaFS():
self._mergeFileTrees(value, node) self._mergeFileTrees(value, node)
else: else:
if key in treeA and treeA[key]=="#REMOVE#" or key in treeB and treeB[key]=="#REMOVE#": if key in treeA and treeA[key]=="#REMOVE#" or key in treeB and treeB[key]=="#REMOVE#":
treeB[key] = "#REMOVE#" del treeB[key]
treeB[key] = value treeB[key] = value
return treeB return treeB
@ -213,71 +221,117 @@ class IotaFS():
newRing = msgpack.dumps(self._fileTree)+self.chainDelimiter newRing = msgpack.dumps(self._fileTree)+self.chainDelimiter
sHash = self.hashState.digest() sHash = self.hashState.digest()
self.blobStore.uploadDataRaw(newRing.encode(), sHash) print("{RING}: "+str(newRing))
payload = newRing
if self.fileCompression:
payload = gzip.compress(payload)
self.blobStore.uploadDataRaw(payload, sHash)
self.hashState.update(newRing) # For every link in the chain, we salt our hashState using the links data self.hashState.update(newRing) # For every link in the chain, we salt our hashState using the links data
def _putBlob(self, data): def _putBlob(self, data):
#TODO: Use secure random provider return self.blobStore.uploadData(data, secrets.token_bytes(64))
return self.blobStore.uploadData(data, str(random.random()*999999999999).encode())
def _getBlob(self, sHash): def _getBlob(self, sHash):
return self.blobStore.getData(sHash) data = self.blobStore.getData(sHash)
return data
def _fetchFile(self, sHash): def _fetchFile(self, sHash):
file = self._getBlob(sHash) file = self._getBlob(sHash)
# file lastFetch lastAccess blob = file
self.cache[sHash] = [file, time.now(), time.now()] if self.fileCompression:
blob = gzip.decompress(blob)
# file lastFetch lastAccess
self.cache[sHash] = [blob, time.time(), time.time()]
return self.cache[sHash] return self.cache[sHash]
def getFile(self, sHash): def getFile(self, sHash):
if sHash==b'':
return [b'', 0, time.time()]
print("/GET/ "+str(sHash)+" <- ")
if sHash in self.cache: if sHash in self.cache:
# TODO: maybe update if to old? # TODO: maybe update if to old?
self.cache[sHash][2] = time.now() self.cache[sHash][2] = time.time()
return self.cache[sHash] return self.cache[sHash]
else: else:
return self._fetchFile(sHash) return self._fetchFile(sHash)
def putFile(self, file, path): def putFile(self, file, path):
blablabla print("/PUT/ "+str(file)+" -> "+path)
if file==b'':
sHash = b''
else:
blob = file
if self.fileCompression:
blob = gzip.compress(blob)
sHash = self._putBlob(blob)
self.cache[sHash] = [file, time.time(), time.time()]
treeDelta = {}
subTree = treeDelta
for elem in path.split("/")[:-1]:
subTree[elem] = {}
subTree = subTree[elem]
subTree[path.split("/")[-1]] = sHash
self.upsertFileTree(treeDelta)
def mkdir(self, path):
treeDelta = {}
subTree = treeDelta
for elem in path.split("/"):
subTree[elem] = {}
subTree = subTree[elem]
self.upsertFileTree(treeDelta)
return 0
def removeFile(self, path):
treeDelta = {}
subTree = treeDelta
for elem in path.split("/")[:-1]:
subTree[elem] = {}
subTree = subTree[elem]
file = subTree[path.split("/")[-1]]
subTree[path.split("/")[-1]] = "#REMOVE#"
self.upsertFileTree(treeDelta)
return file
def mv(self, old, new):
file = self.removeFile(old)
self.putFile(self, file, new)
class IotaFS_Fuse(LoggingMixIn, Operations): class IotaFS_Fuse(LoggingMixIn, Operations):
def __init__(self, token): def __init__(self, token, fileCompression=True):
self.fs = IotaFS(token) self.fs = IotaFS(token, fileCompression=fileCompression)
self.fileTree = self.fs.getFileTree()
def getSubtree(self, path): def getSubtree(self, path):
subTree = self.fileTree subTree = self.fs.getFileTree()
for elem in path.split("/"): for elem in path[1:].split("/"):
if elem not in subTree: if elem!="":
return False if elem not in subTree:
subTree = subTree[elem] return False
if self.subtreeIsFile(subTree):
# we cannot traverse further, if this is a file
return False
subTree = subTree[elem]
return subTree return subTree
def createFileObj(self, path, fileObj): def createFileObj(self, path, fileObj):
for elem in path.split("/")[:-1]: subTree = self.fileTree
for elem in path[1:].split("/")[:-1]:
if elem not in subTree: if elem not in subTree:
return False return False
subTree = subTree[elem] subTree = subTree[elem]
subTree[path.split("/")[-1]] = fileObj subTree[path.split("/")[-1]] = fileObj
def subtreeIsFile(self, subtree): def subtreeIsFile(self, subtree):
return isinstance(subtree, str) return isinstance(subtree, (bytes, bytearray))
def subtreeExists(self, subtree): def subtreeExists(self, subtree):
return not (subtree == False) return not (subtree == False)
#def chmod(self, path, mode): def create(self, path, mode):
# return self.sftp.chmod(path, mode) print("[#] CREATE "+path)
self.fs.putFile(b'', path[1:])
#def chown(self, path, uid, gid): #return open(path[1:])
# return self.sftp.chown(path, uid, gid) return 0
#def create(self, path, mode):
# f = self.sftp.open(path, 'w')
# f.chmod(mode)
# f.close()
# return 0
#def destroy(self, path): #def destroy(self, path):
# self.sftp.close() # self.sftp.close()
@ -293,69 +347,81 @@ class IotaFS_Fuse(LoggingMixIn, Operations):
st = {} st = {}
# mode decides access permissions and if file object is a directory (stat.S_IFDIR), file (stat.S_IFREG) or a special file # mode decides access permissions and if file object is a directory (stat.S_IFDIR), file (stat.S_IFREG) or a special file
if self.subtreeIsFile(subTree): if self.subtreeIsFile(subTree):
st['st_mode'] = 777 | stat.S_IFREG st['st_mode'] = 0o744 | stat.S_IFREG
else: else:
st['st_mode'] = 777 | stat.S_IFDIR st['st_mode'] = 0o744 | stat.S_IFDIR
st['st_ino'] = 0 #st['st_ino'] = 0
st['st_dev'] = 0 #st['st_dev'] = 0
st['st_nlink'] = 1 st['st_nlink'] = 1
st['st_uid'] = os.getuid() #file object's user id st['st_uid'] = os.getuid() #file object's user id
st['st_gid'] = os.getgid() #file object's group id st['st_gid'] = os.getgid() #file object's group id
if fh: if fh and False:
file, path, sHash, lastFetch, lastAccess = fh file, path, sHash, lastFetch, lastAccess = fh
st["st_size"] = len(file) st["st_size"] = len(file)
st['st_atime'] = lastAccess st['st_atime'] = lastAccess
st['st_mtime'] = lastFetch st['st_mtime'] = lastFetch
st['st_ctime'] = 0 st['st_ctime'] = 0
else: else:
st['st_size'] = 0 # 0 Byte lol st['st_size'] = 1024*1024*1024 # 1 Byte lol
st['st_atime'] = now #last access time in seconds st['st_atime'] = 0 #last access time in seconds
st['st_mtime'] = now #last modified time in seconds st['st_mtime'] = 0 #last modified time in seconds
st['st_ctime'] = 0 # very old file st['st_ctime'] = 0 # very old file
# TODO: Actuall real value # TODO: Actuall real value
block_size = 512 block_size = 512
st['st_blocks'] = (int) ((st['st_size'] + block_size-1) / block_size) st['st_blocks'] = (int) ((st['st_size'] + block_size-1) / block_size)
return st return st
#def mkdir(self, path, mode): def mkdir(self, path, mode):
# return self.sftp.mkdir(path, mode) print("[#] MKDIR "+path)
self.fs.mkdir(path[1:])
return 0
def read(self, path2, size, offset, fh): def read(self, path, size, offset, fh):
print("[#] WRITE "+path2) print("[#] READ "+path)
file, path, sHash, lastFetch, lastAccess = fh file, path2, sHash, lastFetch, lastAccess = self.openFile(path)
if path!=path2: if path!=path2:
print(path+"!="+path2)
return FuseOSError(ENOENT) return FuseOSError(ENOENT)
return file[offset : offset+size] return file[offset : offset+size]
def readdir(self, path, fh): def readdir(self, path, fh):
print("[#] READDIR "+path) print("[#] READDIR "+path)
l = ['.', '..'] subTree = self.getSubtree(path)
for elem in self.getSubtree(path): if self.subtreeIsFile(subTree):
l.append(elem.encode('utf-8')) # We cant list a file!
return FuseOSError(ENOENT)
pprint(subTree)
l = [".", ".."]
for elem in subTree:
l.append(elem)
return l return l
#def rename(self, old, new): def rename(self, old, new):
# return self.sftp.rename(old, new) self.fs.mv(old,new)
return 0
#def rmdir(self, path): def rmdir(self, path):
# return self.sftp.rmdir(path) self.fs.removeFile(path)
return 0
def write(self, path2, data, offset, fh): def write(self, path, data, offset, fh):
print("[#] WRITE "+path2) print("[#] WRITE "+path)
file, path, sHash, lastFetch, lastAccess = fh file, path2, sHash, lastFetch, lastAccess = self.openFile(path)
if path!=path2: if path!=path2:
print(path+"!="+path2)
return FuseOSError(ENOENT) return FuseOSError(ENOENT)
raw = data.encode() raw = data
file[:offset] + raw + file[offset+len(raw):] file = file[:offset] + raw + file[offset+len(raw):]
self.fs.putFile(file, path[1:])
print("Write successfull")
return len(raw) return len(raw)
def open(self, path, flags): def openFile(self, path):
print("[#] OPEN "+path)
subTree = self.getSubtree(path) subTree = self.getSubtree(path)
if subTree == False: if subTree == False:
pass raise FuseOSError(ENOENT)
else: else:
if not self.subtreeIsFile(subTree): if not self.subtreeIsFile(subTree):
# cannot open a dir # cannot open a dir
@ -364,6 +430,12 @@ class IotaFS_Fuse(LoggingMixIn, Operations):
file, lastFetch, lastAccess = self.fs.getFile(sHash) file, lastFetch, lastAccess = self.fs.getFile(sHash)
return (file, path, sHash, lastFetch, lastAccess) return (file, path, sHash, lastFetch, lastAccess)
def open(self, path, flags):
return 0
def release(self, path, fh):
return 0
if __name__ == '__main__': if __name__ == '__main__':
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()