everything works! (except for compression)
This commit is contained in:
parent
07dd1fb4e2
commit
d0ac89b622
BIN
__pycache__/main.cpython-38.pyc
Normal file
BIN
__pycache__/main.cpython-38.pyc
Normal file
Binary file not shown.
226
main.py
226
main.py
@ -23,6 +23,8 @@ from errno import ENOENT
|
||||
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
|
||||
import stat
|
||||
import os
|
||||
import gzip
|
||||
import secrets
|
||||
|
||||
class IotaFS_BlobStore():
|
||||
def __init__(self, api=None):
|
||||
@ -31,7 +33,7 @@ class IotaFS_BlobStore():
|
||||
else:
|
||||
self.api = api
|
||||
|
||||
def genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1):
|
||||
def _genBundles(self, data, addrIter, lenPerTx = 2187, txPerBundle = 1):
|
||||
msg = TryteString.from_bytes(data)
|
||||
bundles = []
|
||||
nextAddr = addrIter.__next__()
|
||||
@ -59,7 +61,7 @@ class IotaFS_BlobStore():
|
||||
)
|
||||
return bundles
|
||||
|
||||
def sendBundles(self, bundles):
|
||||
def _sendBundles(self, bundles):
|
||||
bundleRets = []
|
||||
for i,bundle in enumerate(bundles):
|
||||
print(str(int(i/len(bundles)*100))+"%")
|
||||
@ -84,8 +86,8 @@ class IotaFS_BlobStore():
|
||||
cipher = AES.new(sHash[:16], AES.MODE_CBC, sHash[22:][:16])
|
||||
ct_bytes = cipher.encrypt(pad(data, AES.block_size))
|
||||
addrIter = AddressGenerator(Seed(trSeed)).create_iterator(start = 0, step = 1)
|
||||
bundles = self.genBundles(ct_bytes, addrIter)
|
||||
self.sendBundles(bundles)
|
||||
bundles = self._genBundles(ct_bytes, addrIter)
|
||||
self._sendBundles(bundles)
|
||||
|
||||
def uploadTxt(self, txt, secret):
|
||||
data = str.encode(txt)
|
||||
@ -107,7 +109,7 @@ class IotaFS_BlobStore():
|
||||
for tx in bundle.transactions:
|
||||
tryteMsg+=str(tx.signature_message_fragment)
|
||||
if tryteMsg == "":
|
||||
return ""
|
||||
return b''
|
||||
tryteStr = TryteString(tryteMsg.rstrip("9"))
|
||||
try:
|
||||
ct_bytes = tryteStr.as_bytes()
|
||||
@ -136,16 +138,22 @@ class IotaFS_BlobStore():
|
||||
f.write(y)
|
||||
|
||||
class IotaFS():
|
||||
def __init__(self, token):
|
||||
def __init__(self, token, fileCompression=False):
|
||||
self.api = Iota('https://nodes.thetangle.org:443', local_pow=True)
|
||||
self.blobStore = IotaFS_BlobStore(self.api)
|
||||
#self.token = token
|
||||
self.fileCompression = fileCompression
|
||||
self.hashState = hashlib.sha3_384()
|
||||
genesis = "This is the genesis block. lol."
|
||||
if self.fileCompression:
|
||||
raise Exception("Compression does not work currently")
|
||||
genesis += "#FILE COMPRESSION#"
|
||||
self.hashState.update(genesis.encode())
|
||||
self.hashState.update(token.encode())
|
||||
self._fileTree = {}
|
||||
self.lastBlockIncomplete = False
|
||||
self.incompleteBlockRescanTimeout = 5
|
||||
self.chainDelimiter = "#CHAIN_DELIM#"
|
||||
self.chainDelimiter = "#IOTA_FS#".encode()
|
||||
self.cache = {}
|
||||
|
||||
self._fetchFileTree()
|
||||
@ -155,34 +163,34 @@ class IotaFS():
|
||||
self._fetchFileTree()
|
||||
return copy.deepcopy(self._fileTree)
|
||||
|
||||
async def fileTreeFetchLoop(self, interval=10):
|
||||
while True:
|
||||
self._fetchFileTree()
|
||||
await asyncio.sleep(10)
|
||||
|
||||
def _fetchFileTree(self):
|
||||
print("[<] Fetching FileTree")
|
||||
chain = ""
|
||||
chain = bytes()
|
||||
while True:
|
||||
print("[<] Fetching FileTree-ChainBlock")
|
||||
sHash = self.hashState.digest()
|
||||
block = self._getBlob(sHash)
|
||||
if block=="":
|
||||
data = block
|
||||
if self.fileCompression:
|
||||
data = gzip.decompress(data)
|
||||
if data==b'':
|
||||
print("[-] Last Block Received")
|
||||
break
|
||||
self.hashState.update(block)
|
||||
chain+=block
|
||||
if chain=="":
|
||||
print("[.] FileTree succesfully fetched: [EMPTY FILE TREE]")
|
||||
chain+=data
|
||||
if chain==b'':
|
||||
print("[.] FileTree succesfully fetched: [NO UPDATES]")
|
||||
return
|
||||
if chain.endswith(self.chainDelimiter):
|
||||
curRing = chain.split(self.chainDelimiter)[-1]
|
||||
curRing = chain.split(self.chainDelimiter)[-2]
|
||||
self.lastBlockIncomplete = False
|
||||
else:
|
||||
print("[-] Last Block was incomplete; refetching...")
|
||||
self.lastBlockIncomplete = True
|
||||
time.sleep(self.incompleteBlockRescanTimeout)
|
||||
self._fetchFileTree()
|
||||
return
|
||||
print("{RING}: "+str(curRing))
|
||||
|
||||
self._fileTree = msgpack.loads(curRing)
|
||||
print("[.] FileTree succesfully fetched: ")
|
||||
@ -200,7 +208,7 @@ class IotaFS():
|
||||
self._mergeFileTrees(value, node)
|
||||
else:
|
||||
if key in treeA and treeA[key]=="#REMOVE#" or key in treeB and treeB[key]=="#REMOVE#":
|
||||
treeB[key] = "#REMOVE#"
|
||||
del treeB[key]
|
||||
treeB[key] = value
|
||||
|
||||
return treeB
|
||||
@ -213,71 +221,117 @@ class IotaFS():
|
||||
|
||||
newRing = msgpack.dumps(self._fileTree)+self.chainDelimiter
|
||||
sHash = self.hashState.digest()
|
||||
self.blobStore.uploadDataRaw(newRing.encode(), sHash)
|
||||
print("{RING}: "+str(newRing))
|
||||
payload = newRing
|
||||
if self.fileCompression:
|
||||
payload = gzip.compress(payload)
|
||||
self.blobStore.uploadDataRaw(payload, sHash)
|
||||
self.hashState.update(newRing) # For every link in the chain, we salt our hashState using the links data
|
||||
|
||||
def _putBlob(self, data):
|
||||
#TODO: Use secure random provider
|
||||
return self.blobStore.uploadData(data, str(random.random()*999999999999).encode())
|
||||
return self.blobStore.uploadData(data, secrets.token_bytes(64))
|
||||
|
||||
def _getBlob(self, sHash):
|
||||
return self.blobStore.getData(sHash)
|
||||
data = self.blobStore.getData(sHash)
|
||||
return data
|
||||
|
||||
def _fetchFile(self, sHash):
|
||||
file = self._getBlob(sHash)
|
||||
# file lastFetch lastAccess
|
||||
self.cache[sHash] = [file, time.now(), time.now()]
|
||||
blob = file
|
||||
if self.fileCompression:
|
||||
blob = gzip.decompress(blob)
|
||||
# file lastFetch lastAccess
|
||||
self.cache[sHash] = [blob, time.time(), time.time()]
|
||||
return self.cache[sHash]
|
||||
|
||||
def getFile(self, sHash):
|
||||
if sHash==b'':
|
||||
return [b'', 0, time.time()]
|
||||
print("/GET/ "+str(sHash)+" <- ")
|
||||
if sHash in self.cache:
|
||||
# TODO: maybe update if to old?
|
||||
self.cache[sHash][2] = time.now()
|
||||
self.cache[sHash][2] = time.time()
|
||||
return self.cache[sHash]
|
||||
else:
|
||||
return self._fetchFile(sHash)
|
||||
|
||||
def putFile(self, file, path):
|
||||
blablabla
|
||||
print("/PUT/ "+str(file)+" -> "+path)
|
||||
if file==b'':
|
||||
sHash = b''
|
||||
else:
|
||||
blob = file
|
||||
if self.fileCompression:
|
||||
blob = gzip.compress(blob)
|
||||
sHash = self._putBlob(blob)
|
||||
self.cache[sHash] = [file, time.time(), time.time()]
|
||||
treeDelta = {}
|
||||
subTree = treeDelta
|
||||
for elem in path.split("/")[:-1]:
|
||||
subTree[elem] = {}
|
||||
subTree = subTree[elem]
|
||||
subTree[path.split("/")[-1]] = sHash
|
||||
self.upsertFileTree(treeDelta)
|
||||
|
||||
def mkdir(self, path):
|
||||
treeDelta = {}
|
||||
subTree = treeDelta
|
||||
for elem in path.split("/"):
|
||||
subTree[elem] = {}
|
||||
subTree = subTree[elem]
|
||||
self.upsertFileTree(treeDelta)
|
||||
return 0
|
||||
|
||||
def removeFile(self, path):
|
||||
treeDelta = {}
|
||||
subTree = treeDelta
|
||||
for elem in path.split("/")[:-1]:
|
||||
subTree[elem] = {}
|
||||
subTree = subTree[elem]
|
||||
file = subTree[path.split("/")[-1]]
|
||||
subTree[path.split("/")[-1]] = "#REMOVE#"
|
||||
self.upsertFileTree(treeDelta)
|
||||
return file
|
||||
|
||||
def mv(self, old, new):
|
||||
file = self.removeFile(old)
|
||||
self.putFile(self, file, new)
|
||||
|
||||
class IotaFS_Fuse(LoggingMixIn, Operations):
|
||||
def __init__(self, token):
|
||||
self.fs = IotaFS(token)
|
||||
self.fileTree = self.fs.getFileTree()
|
||||
def __init__(self, token, fileCompression=True):
|
||||
self.fs = IotaFS(token, fileCompression=fileCompression)
|
||||
|
||||
def getSubtree(self, path):
|
||||
subTree = self.fileTree
|
||||
for elem in path.split("/"):
|
||||
if elem not in subTree:
|
||||
return False
|
||||
subTree = subTree[elem]
|
||||
subTree = self.fs.getFileTree()
|
||||
for elem in path[1:].split("/"):
|
||||
if elem!="":
|
||||
if elem not in subTree:
|
||||
return False
|
||||
if self.subtreeIsFile(subTree):
|
||||
# we cannot traverse further, if this is a file
|
||||
return False
|
||||
subTree = subTree[elem]
|
||||
return subTree
|
||||
|
||||
def createFileObj(self, path, fileObj):
|
||||
for elem in path.split("/")[:-1]:
|
||||
subTree = self.fileTree
|
||||
for elem in path[1:].split("/")[:-1]:
|
||||
if elem not in subTree:
|
||||
return False
|
||||
subTree = subTree[elem]
|
||||
subTree[path.split("/")[-1]] = fileObj
|
||||
|
||||
def subtreeIsFile(self, subtree):
|
||||
return isinstance(subtree, str)
|
||||
return isinstance(subtree, (bytes, bytearray))
|
||||
|
||||
def subtreeExists(self, subtree):
|
||||
return not (subtree == False)
|
||||
|
||||
#def chmod(self, path, mode):
|
||||
# return self.sftp.chmod(path, mode)
|
||||
|
||||
#def chown(self, path, uid, gid):
|
||||
# return self.sftp.chown(path, uid, gid)
|
||||
|
||||
#def create(self, path, mode):
|
||||
# f = self.sftp.open(path, 'w')
|
||||
# f.chmod(mode)
|
||||
# f.close()
|
||||
# return 0
|
||||
def create(self, path, mode):
|
||||
print("[#] CREATE "+path)
|
||||
self.fs.putFile(b'', path[1:])
|
||||
#return open(path[1:])
|
||||
return 0
|
||||
|
||||
#def destroy(self, path):
|
||||
# self.sftp.close()
|
||||
@ -293,69 +347,81 @@ class IotaFS_Fuse(LoggingMixIn, Operations):
|
||||
st = {}
|
||||
# mode decides access permissions and if file object is a directory (stat.S_IFDIR), file (stat.S_IFREG) or a special file
|
||||
if self.subtreeIsFile(subTree):
|
||||
st['st_mode'] = 777 | stat.S_IFREG
|
||||
st['st_mode'] = 0o744 | stat.S_IFREG
|
||||
else:
|
||||
st['st_mode'] = 777 | stat.S_IFDIR
|
||||
st['st_ino'] = 0
|
||||
st['st_dev'] = 0
|
||||
st['st_mode'] = 0o744 | stat.S_IFDIR
|
||||
#st['st_ino'] = 0
|
||||
#st['st_dev'] = 0
|
||||
st['st_nlink'] = 1
|
||||
st['st_uid'] = os.getuid() #file object's user id
|
||||
st['st_gid'] = os.getgid() #file object's group id
|
||||
if fh:
|
||||
if fh and False:
|
||||
file, path, sHash, lastFetch, lastAccess = fh
|
||||
st["st_size"] = len(file)
|
||||
st['st_atime'] = lastAccess
|
||||
st['st_mtime'] = lastFetch
|
||||
st['st_ctime'] = 0
|
||||
else:
|
||||
st['st_size'] = 0 # 0 Byte lol
|
||||
st['st_atime'] = now #last access time in seconds
|
||||
st['st_mtime'] = now #last modified time in seconds
|
||||
st['st_size'] = 1024*1024*1024 # 1 Byte lol
|
||||
st['st_atime'] = 0 #last access time in seconds
|
||||
st['st_mtime'] = 0 #last modified time in seconds
|
||||
st['st_ctime'] = 0 # very old file
|
||||
# TODO: Actuall real value
|
||||
block_size = 512
|
||||
st['st_blocks'] = (int) ((st['st_size'] + block_size-1) / block_size)
|
||||
return st
|
||||
|
||||
#def mkdir(self, path, mode):
|
||||
# return self.sftp.mkdir(path, mode)
|
||||
def mkdir(self, path, mode):
|
||||
print("[#] MKDIR "+path)
|
||||
self.fs.mkdir(path[1:])
|
||||
return 0
|
||||
|
||||
def read(self, path2, size, offset, fh):
|
||||
print("[#] WRITE "+path2)
|
||||
file, path, sHash, lastFetch, lastAccess = fh
|
||||
def read(self, path, size, offset, fh):
|
||||
print("[#] READ "+path)
|
||||
file, path2, sHash, lastFetch, lastAccess = self.openFile(path)
|
||||
if path!=path2:
|
||||
print(path+"!="+path2)
|
||||
return FuseOSError(ENOENT)
|
||||
|
||||
return file[offset : offset+size]
|
||||
|
||||
def readdir(self, path, fh):
|
||||
print("[#] READDIR "+path)
|
||||
l = ['.', '..']
|
||||
for elem in self.getSubtree(path):
|
||||
l.append(elem.encode('utf-8'))
|
||||
subTree = self.getSubtree(path)
|
||||
if self.subtreeIsFile(subTree):
|
||||
# We cant list a file!
|
||||
return FuseOSError(ENOENT)
|
||||
pprint(subTree)
|
||||
l = [".", ".."]
|
||||
for elem in subTree:
|
||||
l.append(elem)
|
||||
return l
|
||||
|
||||
#def rename(self, old, new):
|
||||
# return self.sftp.rename(old, new)
|
||||
def rename(self, old, new):
|
||||
self.fs.mv(old,new)
|
||||
return 0
|
||||
|
||||
#def rmdir(self, path):
|
||||
# return self.sftp.rmdir(path)
|
||||
def rmdir(self, path):
|
||||
self.fs.removeFile(path)
|
||||
return 0
|
||||
|
||||
def write(self, path2, data, offset, fh):
|
||||
print("[#] WRITE "+path2)
|
||||
file, path, sHash, lastFetch, lastAccess = fh
|
||||
def write(self, path, data, offset, fh):
|
||||
print("[#] WRITE "+path)
|
||||
file, path2, sHash, lastFetch, lastAccess = self.openFile(path)
|
||||
if path!=path2:
|
||||
print(path+"!="+path2)
|
||||
return FuseOSError(ENOENT)
|
||||
|
||||
raw = data.encode()
|
||||
file[:offset] + raw + file[offset+len(raw):]
|
||||
raw = data
|
||||
file = file[:offset] + raw + file[offset+len(raw):]
|
||||
|
||||
self.fs.putFile(file, path[1:])
|
||||
print("Write successfull")
|
||||
return len(raw)
|
||||
|
||||
def open(self, path, flags):
|
||||
print("[#] OPEN "+path)
|
||||
def openFile(self, path):
|
||||
subTree = self.getSubtree(path)
|
||||
if subTree == False:
|
||||
pass
|
||||
raise FuseOSError(ENOENT)
|
||||
else:
|
||||
if not self.subtreeIsFile(subTree):
|
||||
# cannot open a dir
|
||||
@ -364,6 +430,12 @@ class IotaFS_Fuse(LoggingMixIn, Operations):
|
||||
file, lastFetch, lastAccess = self.fs.getFile(sHash)
|
||||
return (file, path, sHash, lastFetch, lastAccess)
|
||||
|
||||
def open(self, path, flags):
|
||||
return 0
|
||||
|
||||
def release(self, path, fh):
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user