419 lines
14 KiB
Python
419 lines
14 KiB
Python
#import pyfuse3
|
|
|
|
# TODO: Milestones always written into new block
|
|
# TODO: Implement File COW (except for append) (+ version / token updates caused by this)
|
|
# TODO: ? Stop using tokens for dirs, use hashed name + parent token
|
|
# TODO: ? Stop using tokens for files, use hashed name + short ?version-number + parent token instead
|
|
# TODO: Store version-numbers for files in parent-dir
|
|
# TODO: Chane Milestone-Format from [stone,stone,...] to [[dirStne,dirStone,...],[fileStone,,...]] and dont save type for the stones
|
|
# TODO: Move iota-push-code from TangleBlob to chunk? and perform when sealing?
|
|
# TODO: When unmounting walk throught tree and seal all blobs
|
|
|
|
from iota import Iota, ProposedTransaction, Address, TryteString, Tag
|
|
from iota.crypto.addresses import AddressGenerator
|
|
from iota.crypto.types import Seed
|
|
from iota.codecs import TrytesDecodeError
|
|
|
|
from Crypto.Cipher import AES
|
|
from Crypto.Util.Padding import pad, unpad
|
|
|
|
import math
|
|
from pprint import pprint
|
|
import hashlib
|
|
import sys
|
|
import random
|
|
import time
|
|
|
|
import msgpack
|
|
import asyncio
|
|
import copy
|
|
|
|
import gzip
|
|
import secrets
|
|
|
|
CHUNKSIZE = 2187
|
|
|
|
def log(txt):
|
|
print("[-] "+str(txt))
|
|
|
|
class Atom():
|
|
def __init__(self, milestone: bool, cont, name: str = None) -> None:
|
|
self.milestone = milestone
|
|
self.delta = not milestone
|
|
self.name = name
|
|
self.cont = cont
|
|
|
|
def dump(self):
|
|
if self.milestone:
|
|
return msgpack.dumps([True, self.cont])
|
|
else:
|
|
return msgpack.dumps([False, self.name, self.cont])
|
|
|
|
class BlobChunk():
|
|
def __init__(self, data: bytes = b'', sealed: bool = False) -> None:
|
|
self.data = data
|
|
self.sealed = sealed
|
|
|
|
def getData(self) -> bytes:
|
|
return self.data
|
|
|
|
def append(self, data: bytes) -> None:
|
|
if len(data)+len(self.data) > CHUNKSIZE:
|
|
raise Exception("That to big!")
|
|
self.data += data
|
|
if len(self.data) == CHUNKSIZE:
|
|
self.seal()
|
|
|
|
def getBytesLeft(self) -> int:
|
|
if self.sealed:
|
|
return 0
|
|
return CHUNKSIZE - len(self.data)
|
|
|
|
def seal(self) -> None:
|
|
self.sealed = True
|
|
|
|
def isSealed(self) -> bool:
|
|
return self.sealed
|
|
|
|
class TangleBlob():
|
|
def __init__(self, token: bytes, iotaApi: Iota) -> None:
|
|
self.token = token
|
|
self.iotaApi = iotaApi
|
|
self.preChunks = 0
|
|
self.chunks = []
|
|
m = hashlib.sha3_512()
|
|
m.update(self.token)
|
|
trSeed = TryteString.from_bytes(m.digest())[:81]
|
|
self.adressGen = AddressGenerator(Seed(trSeed))
|
|
self.fetched = False
|
|
self.pushedNum = 0
|
|
|
|
def _requireFetched(self):
|
|
if not self.fetched:
|
|
self.fetch()
|
|
|
|
def _getKey(self, chunkNum: int) -> bytes:
|
|
m = hashlib.sha3_384()
|
|
m.update(self.token)
|
|
m.update(chunkNum.to_bytes(8, "little")) # 64 bits should be enought...
|
|
m.update(self.token)
|
|
return m.digest()
|
|
|
|
def _genBundle(self, data, addr) -> str:
|
|
txMsg = TryteString.from_bytes(data)
|
|
trans = ProposedTransaction(
|
|
address = addr,
|
|
value = 0,
|
|
tag = Tag("IOTAFS"),
|
|
message = txMsg
|
|
)
|
|
return self.iotaApi.prepare_transfer(
|
|
transfers = [trans],
|
|
inputs = [addr]
|
|
)['trytes']
|
|
|
|
def _dumpChunk(self, chunkNum: int) -> str:
|
|
key = self._getKey(chunkNum + self.preChunks)
|
|
data = self.chunks[chunkNum].getData()
|
|
cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16])
|
|
ct_bytes = cipher.encrypt(pad(data, AES.block_size))
|
|
addr = self.adressGen.get_addresses(start=chunkNum + self.preChunks, count=1)[0]
|
|
return self._genBundle(ct_bytes, addr)
|
|
|
|
def append(self, data: bytes, newBlock: bool = False) -> None:
|
|
self._requireFetched()
|
|
if len(self.chunks) and not newBlock:
|
|
bytesLeft = self.chunks[-1].getBytesLeft()
|
|
if bytesLeft:
|
|
leftChunk = data[:bytesLeft]
|
|
data = data[bytesLeft:]
|
|
self.chunks[-1].append(leftChunk)
|
|
while len(data):
|
|
chunk = data[:CHUNKSIZE]
|
|
self.chunks.append(BlobChunk(chunk))
|
|
data = data[CHUNKSIZE:]
|
|
|
|
def getChunkLen(self) -> int:
|
|
return self.preChunks + len(self.chunks)
|
|
|
|
def getSize(self) -> int:
|
|
if len(self.chunks):
|
|
return self.getChunkLen()*CHUNKSIZE - self.chunks[-1].getBytesLeft()
|
|
return self.preChunks
|
|
|
|
def read(self) -> bytes:
|
|
data = b''
|
|
for chunk in self.chunks:
|
|
data += chunk.getData()
|
|
return data
|
|
|
|
def _dump(self) -> str:
|
|
self.chunks[-1].seal()
|
|
data = ""
|
|
for c in range(len(self.chunks)-self.pushedNum):
|
|
num = c + self.pushedNum
|
|
data += self._dumpChunk(num) # num is without preChunks
|
|
self.pushedNum = self.getChunkLen()
|
|
return data
|
|
|
|
def fetch(self) -> None:
|
|
skipChunks = self.preChunks
|
|
chunkNum = self.getChunkLen() + skipChunks
|
|
while True:
|
|
key = self._getKey(chunkNum)
|
|
cipher = AES.new(key[16:][:16], AES.MODE_CBC, key[:16])
|
|
addr = self.adressGen.get_addresses(start=chunkNum, count=1)[0]
|
|
txHash = self.iotaApi.find_transactions(tags=[Tag("IOTAFS")], addresses=[addr])["hashes"]
|
|
if len(txHash)==0:
|
|
break
|
|
bundles = self.api.get_bundles(txHash[0])["bundles"]
|
|
for bundle in bundles:
|
|
for tx in bundle.transactions:
|
|
# TODO: Can we just strip the 9s and call it a day?
|
|
tryteStr = TryteString(str(tx.signature_message_fragment).rstrip("9"))
|
|
try:
|
|
ct_bytes = tryteStr.as_bytes()
|
|
except TrytesDecodeError:
|
|
ct_bytes = (tryteStr+"9").as_bytes()
|
|
self.chunks.append(BlobChunk(unpad(cipher.decrypt(ct_bytes), AES.block_size), True))
|
|
chunkNum += 1
|
|
self.pushedNum = len(self.chunks)
|
|
self.fetched = True
|
|
self._afterFetch()
|
|
|
|
def _afterFetch(self) -> None:
|
|
return
|
|
|
|
def genToken(self) -> bytes:
|
|
return secrets.token_bytes(32)
|
|
|
|
def sealLastChunk(self) -> None:
|
|
self.chunks[-1].seal()
|
|
|
|
class TangleFileTreeElement(TangleBlob):
|
|
def __init__(self, name: str, lastMilestoneIndex: int, parent, iotaApi: Iota) -> None:
|
|
if isinstance(parent, bytes):
|
|
self.token = hashlib.sha3_384(parent + name.encode()).digest()
|
|
else:
|
|
self.token = hashlib.sha3_384(parent.token + name.encode()).digest()
|
|
super(TangleFileTreeElement, self).__init__(self.token, iotaApi)
|
|
self.name = name
|
|
self.inodes = {}
|
|
self.parent = parent
|
|
self.milestoneIndex = lastMilestoneIndex
|
|
self.preChunks = self.milestoneIndex
|
|
|
|
def _afterFetch(self) -> None:
|
|
raw = self.read()
|
|
if raw==b'':
|
|
return
|
|
unpacker = msgpack.Unpacker(raw=True)
|
|
unpacker.feed(raw)
|
|
for i, elem in enumerate(reversed(unpacker)):
|
|
if elem[0]:
|
|
# Is a milestone
|
|
# TODO: Update our known milestoneIndex, if we find one
|
|
# might have to rewrite .fetch() and merge it here...
|
|
self.milestoneIndex = self.getChunkLen()
|
|
self._applyMilestone(elem[1])
|
|
break
|
|
else:
|
|
if elem[1] in self.inodes: #name
|
|
self.inodes[elem[1]].applyDelta(elem[2])
|
|
else:
|
|
# new file
|
|
self.inodes[elem[1]] = Inode(elem[1], self.iotaApi)
|
|
self.inodes[elem[1]].applyDelta(elem[2])
|
|
|
|
def _getSkipChunks(self):
|
|
return
|
|
|
|
def _applyMilestone(self, milestone) -> None:
|
|
self.inodes = {}
|
|
for stone in milestone:
|
|
atom = Atom()
|
|
atom.load(stone)
|
|
self.inodes[atom.name] = atom
|
|
|
|
def _applyAtom(self, atom: Atom) -> None:
|
|
if atom.name in self.inodes:
|
|
self.inodes[atom.name].applyAtom(atom)
|
|
else:
|
|
cont = atom.cont
|
|
type = ["dir","file"][cont[b't']]
|
|
inode = Inode(atom.name, self.iotaApi, self, type)
|
|
self.inodes[atom.name] = inode
|
|
|
|
def _newAtom(self, atom: Atom) -> None:
|
|
self.append(atom.dump())
|
|
|
|
def getNameList(self):
|
|
self._requireFetched()
|
|
return list(self.inodes.keys())
|
|
|
|
def getInode(self, name: str) -> Atom:
|
|
self._requireFetched()
|
|
return self.inodes[name]
|
|
|
|
def mkdir(self, name: str) -> bool:
|
|
self._requireFetched()
|
|
if name in self.getNameList():
|
|
return False
|
|
inode = Inode(name, self.iotaApi, self, "dir")
|
|
atom = inode.change(milestoneIndex=0)
|
|
self._newAtom(atom)
|
|
self.inodes[name] = inode
|
|
return True
|
|
|
|
def mkfile(self, name: str) -> bool:
|
|
self._requireFetched()
|
|
if name in self.getNameList():
|
|
return False
|
|
file = Inode(name, self.iotaApi, self, "file")
|
|
atom = file.change(size=0, milestoneIndex=0, hash=b'NULL')
|
|
self._newAtom(atom)
|
|
self._applyAtom(atom)
|
|
return True
|
|
|
|
def _updateFileSize(self, name: str, size: int) -> None:
|
|
self._requireFetched()
|
|
self.inodes[name].size = size
|
|
self._newAtom(self.inodes[name])
|
|
|
|
def _updateFileToken(self, name: str, token: bytes, size: int) -> None:
|
|
log("New FileToken for file '"+name+"' registered")
|
|
self._requireFetched()
|
|
self.inodes[name].token = token
|
|
self.inodes[name].size = size
|
|
self._newAtom(self.inodes[name])
|
|
|
|
def performMilestone(self) -> None:
|
|
stones = {}
|
|
for a in self.inodes:
|
|
stones[a] = self.inodes[a].toStone()
|
|
self.atomStack = 0
|
|
# TODO: Delimiter ?
|
|
# TODO: compression ?
|
|
milestoneAtom = Atom(True, stones)
|
|
data = milestoneAtom.dump()
|
|
self.milestoneIndex = self.getSize()
|
|
if self.parent!=None:
|
|
self.parent._updateChildMilestone(self.name, self.milestoneIndex)
|
|
self.append(data, True)
|
|
# inform parent about milestone (when merged)
|
|
|
|
def _updateChildMilestone(self, name: str, milestoneIndex: int):
|
|
self.inodes[name].milestoneIndex = milestoneIndex
|
|
self._newAtom(self.inodes[name])
|
|
|
|
class TangleFile():
|
|
def __init__(self, name: str, parent: TangleFileTreeElement, iotaApi: Iota) -> None:
|
|
self.api = iotaApi
|
|
self.name = name
|
|
self.parent = parent
|
|
self.reflexiveAtom = parent.inodes[name]
|
|
self.size = self.reflexiveAtom.size
|
|
self.hash = self.reflexiveAtom.hash
|
|
self.token = hashlib.sha3_384(b'f' + parent.getRef().token + self.hash).digest()
|
|
self.blob = TangleBlob(self.token, iotaApi)
|
|
|
|
def write(self, offset: int, data: bytes):
|
|
if offset == self.size:
|
|
self.append(data)
|
|
else:
|
|
self.token = self.genToken()
|
|
oldData = self.blob.read()
|
|
newData = oldData[:offset] + data + oldData[offset+len(data):]
|
|
self.blob = TangleBlob(self.token)
|
|
self.blob.append(newData)
|
|
self.size = self.blob.getSize()
|
|
self.parent._updateFileToken(self.name, self.token, self.size)
|
|
|
|
class Inode():
|
|
def __init__(self, name: str, iotaApi: Iota, parent: TangleFileTreeElement = None, type: str = None) -> None:
|
|
self.parent = parent
|
|
self.name = name
|
|
self.type = type
|
|
self.ref = None
|
|
self.iotaApi = iotaApi
|
|
|
|
def setType(self, type: str) -> None:
|
|
self.type = type
|
|
|
|
def change(self, size: int=None, hash: int=None, milestoneIndex: int=None) -> Atom:
|
|
delta = {}
|
|
delta[b't'] = (self.type=="file")
|
|
if size!=None:
|
|
self.size = size
|
|
delta[b's'] = size
|
|
if hash!=None:
|
|
self.hash = hash
|
|
delta[b'h'] = hash
|
|
if milestoneIndex!=None:
|
|
self.milestoneIndex = milestoneIndex
|
|
delta[b'm'] = milestoneIndex
|
|
return Atom(False, delta, self.name)
|
|
|
|
def applyAtom(self, atom: Atom) -> None:
|
|
if atom.name != self.name:
|
|
raise Exception("Cannot apply atom ment for a different inode (names differ)")
|
|
if atom.milestone:
|
|
stones = atom.cont
|
|
if self.name in stones:
|
|
self.applyAtom(Atom(False, stones[self.name], self.name))
|
|
else:
|
|
if (self.type=="file") != atom.cont[b't']:
|
|
raise Exception("I am a "+self.type+"; this atom is for the other thing")
|
|
delta = atom.cont
|
|
if b's' in delta:
|
|
self.size = delta[b's']
|
|
if b'm' in delta:
|
|
self.milestoneIndex = delta[b'm']
|
|
|
|
def toStone(self) -> None:
|
|
if self.type=="file":
|
|
return [self.size, self.hash]
|
|
else:
|
|
return [self.milestoneIndex]
|
|
|
|
def getRef(self):
|
|
if self.name == "*":
|
|
return None
|
|
if not self.ref:
|
|
if self.type=="dir":
|
|
self.ref = TangleFileTreeElement(self.name, self.milestoneIndex, self.parent, self.iotaApi)
|
|
elif self.type=="file":
|
|
self.ref = TangleFile(self.name, self, self.iotaApi)
|
|
else:
|
|
raise Exception("Cannot get reference of an inode of type "+self.type)
|
|
return self.ref
|
|
|
|
class IotaFs():
|
|
def __init__(self, token) -> None:
|
|
self.api = Iota('https://nodes.thetangle.org:443', local_pow=True)
|
|
# TODO Cache last known milestone-Index of genesis locally
|
|
self.genesis = TangleFileTreeElement("*", 0, token, None, self.api)
|
|
log("Fetching Genesis...")
|
|
self.genesis.fetch()
|
|
log("Retrieving reference to root")
|
|
if self.genesis.getNameList()!=["/"]:
|
|
if len(self.genesis.getNameList()):
|
|
# theres another directory in our genesis chain... WTF?!
|
|
raise Exception("Corrupted Genesis-Chain:"
|
|
+ "Unknown records for no root-directory in Genesis Chain: "+str(self.genesis.getNameList()))
|
|
else:
|
|
# we dont have a root yet, lets create one...
|
|
log("Unable to reference to root: Creating new root")
|
|
self.genesis.mkdir("/")
|
|
log("Successfully Mounted!")
|
|
|
|
|
|
def createNewFile(self, name) -> None:
|
|
pass
|
|
|
|
api = Iota('https://nodes.thetangle.org:443', local_pow=True)
|
|
token = b'testToken'
|
|
genesis = TangleFileTreeElement("*", 0, token, api)
|
|
genesis.mkdir("/")
|
|
root = genesis.inodes["/"].getRef()
|