dodox/CaliGraph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Allow disabling pagerank

Browse Source
This commit is contained in:
Dominik Moritz Roth 2022-02-22 15:22:36 +01:00
parent 01d41f3a82
commit 9a02bdc2a8
1 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
caliGraph.py
View File

@ -427,6 +427,7 @@ def scoreUnread(G, globMu, globStd):
neuralBins['sigma'] = [node['std']] neuralBins['sigma'] = [node['std']]
neuralBins['median'] = [node['median']] neuralBins['median'] = [node['median']]
neuralBins['se'] = [node['se']] neuralBins['se'] = [node['se']]
if 'pagerank_score' in node:
neuralBins['pagerank'] = [node['pagerank_score']] neuralBins['pagerank'] = [node['pagerank_score']]
if 'tgb_rank' in node: if 'tgb_rank' in node:
neuralBins['tgbrank'] = [10/math.ln10(10+node['tgb_rank'])] neuralBins['tgbrank'] = [10/math.ln10(10+node['tgb_rank'])]
@ -1199,7 +1200,7 @@ def findNewBooks(G, books, mu, num=-1, minRecSco=5):
# while batchSize is implemented, we only get a good gonvergence when we disable it (batchSize=-1) # while batchSize is implemented, we only get a good gonvergence when we disable it (batchSize=-1)
# but might be necessary to enable later for a larger libary for better training performance... # but might be necessary to enable later for a larger libary for better training performance...
# maybe try again for 128 books? # maybe try again for 128 books?
def evaluateFitness(books, batchSize=16, debugPrint=False): def evaluateFitness(books, batchSize=16, debugPrint=False, runPagerank=True):
global weights global weights
G = buildBookGraph(books) G = buildBookGraph(books)
graphAddAuthors(G, books) graphAddAuthors(G, books)
@ -1207,6 +1208,7 @@ def evaluateFitness(books, batchSize=16, debugPrint=False):
graphAddTopLists(G, books) graphAddTopLists(G, books)
graphAddSeries(G, books) graphAddSeries(G, books)
graphAddTags(G, books) graphAddTags(G, books)
if runPagerank:
runPagerank(G) runPagerank(G)
ratedBooks = [n for n in list(G.nodes) if 'rating' in G.nodes[n] and G.nodes[n]['rating'] != None] ratedBooks = [n for n in list(G.nodes) if 'rating' in G.nodes[n] and G.nodes[n]['rating'] != None]
@ -1216,7 +1218,7 @@ def evaluateFitness(books, batchSize=16, debugPrint=False):
gradient = {} gradient = {}
for w in weights: for w in weights:
gradient[w] = 0 gradient[w] = 0
mu, sigma = genScores(G, books) mu, sigma = genScores(G, books, calcPagerank=runPagerank)
batch = random.sample(ratedBooks, batchSize) if batchSize!=-1 and len(ratedBooks) > batchSize else ratedBooks batch = random.sample(ratedBooks, batchSize) if batchSize!=-1 and len(ratedBooks) > batchSize else ratedBooks
for b in G.nodes: for b in G.nodes:
if b in ratedBooks: if b in ratedBooks:
@ -1248,7 +1250,7 @@ def evaluateFitness(books, batchSize=16, debugPrint=False):
fit = sum(errSq)/len(errSq) + 0.001*regressionLoss fit = sum(errSq)/len(errSq) + 0.001*regressionLoss
return fit, gradient return fit, gradient
def train(initGamma, full=True): def train(initGamma, full=True, noPagerank=False):
global weights global weights
if full: if full:
for wt in weights: for wt in weights:
@ -1257,7 +1259,7 @@ def train(initGamma, full=True):
gamma = initGamma gamma = initGamma
books = loadBooksFromDB() books = loadBooksFromDB()
bestWeights = copy.copy(weights) bestWeights = copy.copy(weights)
mse, gradient = evaluateFitness(books) mse, gradient = evaluateFitness(books, runPagerank=not noPagerank)
delta = math.sqrt(sum(gradient[g]**2 for g in gradient)/len(gradient)) delta = math.sqrt(sum(gradient[g]**2 for g in gradient)/len(gradient))
best_mse = mse best_mse = mse
stagLen = 0 stagLen = 0
@ -1275,7 +1277,7 @@ def train(initGamma, full=True):
weights[wt] += gamma*gradient[wt]/math.sqrt(delta) weights[wt] += gamma*gradient[wt]/math.sqrt(delta)
#else: #else:
# del weights[wt] # del weights[wt]
mse, gradient = evaluateFitness(books) mse, gradient = evaluateFitness(books, runPagerank=not noPagerank)
if mse < last_mse: if mse < last_mse:
gamma = gamma*1.25 gamma = gamma*1.25
else: else:
@ -1332,6 +1334,7 @@ def cliInterface():
parser.add_argument('--v3d', action="store_true") parser.add_argument('--v3d', action="store_true")
parser.add_argument('--imgs', action="store_true") parser.add_argument('--imgs', action="store_true")
parser.add_argument('--perf-test', action="store_true") parser.add_argument('--perf-test', action="store_true")
parser.add_argument('--no-pagerank', action="store_true")
cmds = parser.add_subparsers(required=True, dest='cmd') cmds = parser.add_subparsers(required=True, dest='cmd')
p_rec = cmds.add_parser('recommend', description="TODO", aliases=['rec']) p_rec = cmds.add_parser('recommend', description="TODO", aliases=['rec'])
@ -1371,12 +1374,13 @@ def cliInterface():
args = parser.parse_args() args = parser.parse_args()
if args.perfTest: if args.perf_test:
perfTestCLI(args) perfTestCLI(args)
else: else:
mainCLI(args) mainCLI(args)
def perfTestCLI(args): def perfTestCLI(args):
import time
from pycallgraph import PyCallGraph from pycallgraph import PyCallGraph
from pycallgraph import Config from pycallgraph import Config
from pycallgraph import GlobbingFilter from pycallgraph import GlobbingFilter
@ -1385,18 +1389,18 @@ def perfTestCLI(args):
config.trace_filter = GlobbingFilter(exclude=[ config.trace_filter = GlobbingFilter(exclude=[
"pycallgraph.*", "pycallgraph.*",
]) ])
with PyCallGraph(output=GraphvizOutput(output_file='perfTests/serve_httpd_' + str(int(time.time())) + '.png'), config=config): with PyCallGraph(output=GraphvizOutput(output_file='perfTests/' + str(int(time.time())) + '.png'), config=config):
mainCLI(args) mainCLI(args)
def mainCLI(args): def mainCLI(args):
if args.cmd=="train": if args.cmd=="train":
train(args.g, args.full) train(args.g, args.full, args.no_pagerank)
exit() exit()
bestListT = 'book' bestListT = 'book'
G, books = buildFullGraph(darkMode=args.dark) G, books = buildFullGraph(darkMode=args.dark)
mu, std = genScores(G, books) mu, std = genScores(G, books, calcPagerank=not args.no_pagerank)
if not args.keep_whitepapers: if not args.keep_whitepapers:
removeWhitepapers(G) removeWhitepapers(G)