прикреплённый файл:ejstat.py к странице FrBrGeorge/PythonCopypasteProof

   1 #!/usr/bin/env python3
   2 '''
   3 '''
   4 
   5 # limit: 10240
   6 
   7 import requests
   8 import subprocess
   9 import re
  10 import os
  11 from os.path import join, isfile, basename, dirname
  12 import sys
  13 import io
  14 from itertools import islice
  15 from glob import iglob
  16 from collections import namedtuple, defaultdict
  17 from functools import partial
  18 import json
  19 import time
  20 import cmd
  21 import traceback
  22 import gzip
  23 import zipfile
  24 import autopep8
  25 import multiprocessing
  26 import pickle
  27 import ast
  28 import atexit
  29 import pprint
  30 import shutil
  31 from editdistance import eval as distance
  32 from difflib import unified_diff, HtmlDiff
  33 try:
  34     import pygments
  35     import pygments.lexers
  36     import pygments.formatters
  37     if not all((os.isatty(1), os.getenv("TERM"), os.getenv("TERM") != "dumb")):
  38         raise ModuleNotFoundError("Not a proper terminal")
  39     def highlight(text, lexer, formatter):
  40         return pygments.highlight(text, getattr(pygments.lexers, lexer)(), getattr(pygments.formatters, formatter)(style=pygSTYLE))
  41 except ModuleNotFoundError:
  42     def highlight(text, lexer, formatter):
  43         return(text)
  44 
  45 TMPHTML="/tmp/ejstat.html"
  46 TMPCSV="/tmp/ejstat.csv"
  47 SITE = "ejudge.cs.msu.ru"
  48 urlROLE = "new-master"
  49 UNSITE = "uneex.org"
  50 urlLECT = "LecturesCMC/PythonIntro2020"
  51 CONTEST = "148"
  52 actGET = "152"
  53 actLOGOUT = "74"
  54 dWORK = os.getcwd()
  55 # TODO refactor cached n* → c*
  56 nCRED = "credentials"
  57 nPATHS = "paths"
  58 nLOAD = "download"
  59 nRUNS = "runs.csv"
  60 nAUDIT = "audit"
  61 nDATES = "dates"
  62 nTASKS = "tasks.zip"
  63 nFORMAT = "formatted.zip"
  64 nSOURCE = "source"
  65 nPREP = "prep.pickle"
  66 nHISTORY = "history"
  67 nDIFFHTML = "diff"
  68 nDISTS = "dists"
  69 nCONFIG = "config"
  70 nREPORT = "report"
  71 nATTACHMENTS = "attachments"
  72 namesKEEP = [nCRED, nDATES, nHISTORY, nCONFIG]
  73 namesRESET = [nPATHS, nLOAD, nRUNS, nTASKS, nFORMAT, nPREP, nDISTS]
  74 dayHALF = 7
  75 dayBONUS = 0.25
  76 secDAY = 24*60*60
  77 SKIPSCORE = 0.66
  78 SCORELINE = "CCCCBBBBAAA"
  79 sepFORMAT = "_@_"
  80 zipMETHOD = zipfile.ZIP_DEFLATED
  81 # TODO filter taskskip completely off the account
  82 taskSKIP = ["HelloWorld"]
  83 ADMINS = ["FrBrGeorge"]
  84 distMIN = 7
  85 distMAX = 100
  86 sizeMAXDIFF = (0.5, 2)
  87 sizeMAXCLUST = 7
  88 pygSTYLE = 'native'
  89 utTIME, utFAILS, utSTATUS, utPENALTY, utRUNID, utDIST, utNEIGH = range(7)
  90 
  91 jsondump = partial(json.dump, indent="    ")
  92 
  93 # TODO Config/args?
  94 def loadconfig():
  95     if isfile(cfg := join(_MeDir, nCONFIG)):
  96         with open(cfg) as f:
  97             res = list(json.load(f))
  98             dummies = set(res[0] if len(res)>0 else taskSKIP)
  99             admins = res[1] if len(res)>1 else ADMINS
 100             fairlist = res[2] if len(res)>2 else {}
 101             indiv = res[3] if len(res)>3 else {}
 102         fairset = {k: set(v) for k, v in fairlist.items()}
 103     return dummies, admins, fairset, indiv
 104 
 105 def saveconfig():
 106     Fairlist = {k: list(v) for k, v in Fairset.items()}
 107     with open(cfg := join(_MeDir, nCONFIG), "w") as f:
 108         return jsondump((list(Dummies), Admins, Fairlist, Individual), f)
 109 
 110 def fairs(task, user=None):
 111     if user:
 112         Fairset.setdefault(task, set()).add(user)
 113     else:
 114         return Fairset.get(task, set())
 115 
 116 dVAR=join(dWORK, "var")
 117 URL = f"https://{SITE}/"
 118 _Me = basename(sys.argv[0])
 119 _MeDir = dirname(os.path.realpath(sys.argv[0]))
 120 _Id = f"{_Me}.{SITE}.{CONTEST}"
 121 dCACHE = join(os.environ["HOME"],".cache",_Id)
 122 HFile = None
 123 
 124 Dummies, Admins, Fairset, Individual = loadconfig()
 125 
 126 def dodiff(a, b, n1, n2):
 127     res = list(unified_diff(a.split('\n'), b.split('\n'), n1, n2, lineterm=""))
 128     return res
 129 
 130 def dohtmldiff(a, b, n1, n2):
 131     differ = HtmlDiff()
 132     return differ.make_file(a.split('\n'), b.split('\n'), n1, n2)
 133 
 134 def debug(*ap, **an):
 135     print(*ap, file=sys.stderr, **an)
 136     sys.stderr.flush()
 137 
 138 def date(secs):
 139     return time.strftime("%d %b", time.localtime(secs))
 140 
 141 def ymd(y_m_d):
 142     return time.mktime(time.strptime(y_m_d, '%Y-%m-%d'))
 143 
 144 def Cached(name, rw, mode="t", check=False, **kwargs):
 145     fname = join(dCACHE, name)
 146     if not isfile(fname) or os.stat(fname).st_size==0:
 147         with open(fname, mode+"w") as f:
 148             if not f.tell():
 149                 print(f"Creating {name}")
 150                 rw(f, True, **kwargs)
 151     with open(fname, mode+"r") as f:
 152         return rw(f, False, **kwargs)
 153 
 154 def rwCred(f, new):
 155     if new:
 156         login = input("Login: ") or "scoreviewer"
 157         passwd = input("Password: ") or "password"
 158         return f.write(f"{login}\n{passwd}\n")
 159     else:
 160         return f.read().strip().split('\n')
 161 
 162 def rwRuns(f, new):
 163     global Request
 164     if new:
 165         Admin, Passwd = Cached(nCRED, rwCred)
 166         Request = requests.post(URL+urlROLE, data = {
 167              "login": Admin,
 168               "password": Passwd,
 169               "contest_id": CONTEST,
 170               "role": "1" }
 171               )
 172         uRuns = re.sub(r"[&]action=\d+|$", f"&action={actGET}", Request.url)
 173         rRuns = requests.get(uRuns, cookies=Request.cookies)
 174         return f.write(rRuns.text)
 175     else:
 176         return f.read()
 177 
 178 def rwDownload(f, new):
 179     if new:
 180         return f.write("done")
 181     else:
 182         TaskDates = Cached(nDATES, rwTaskDates)
 183         Runs = Cached(nRUNS, rwRuns)
 184         return Runs, TaskDates
 185 
 186 def processData(lines, dates, sep=";"):
 187     L = lines.strip().split("\n")
 188     names = ["Path"] + L[0].split(sep) + ["junk"]
 189     Runline = namedtuple("Runline", names)
 190     idx = names.index("Run_Id")
 191     Runs = {}
 192     for l in islice(L, 1, None):
 193         line = l.split(sep)
 194         i = line[idx-1]
 195         if i not in Paths:
 196             print(f"New run: {i}", file=sys.stderr)
 197             continue
 198         run = Runline(Paths[i], *l.split(sep))
 199         Runs[run.Run_Id] = run
 200     return Runs, dates
 201 
 202 def rwPaths(f, new):
 203     global Paths
 204     if new:
 205         Paths = {}
 206         for faudit in iglob(join(dVAR,"**",nAUDIT), recursive=True):
 207             with open(faudit) as fa:
 208                 for l in fa:
 209                     cmd, data = l.strip().split(": ", 1)
 210                     if cmd == "Run-id":
 211                         Paths[data] = dirname(faudit)
 212                         break
 213         return jsondump(Paths, f)
 214     else:
 215         return json.load(f)
 216 
 217 def rwTaskDates(f, new):
 218     if new:
 219         TaskDates = {}
 220         U = f"http://{UNSITE}/{urlLECT}" # ?action=raw"
 221         r = requests.get(U+"?action=raw")
 222         for l in r.text.split("\n"):
 223             if res := re.match(r"\|\|.*\[\[(/\d\d_[^|]+).*<<Date.*<<Date[(]([^T]+).*\|\|", l):
 224                 page, date = res.groups()
 225                 rr = requests.get(U+page+"?action=raw")
 226                 for ll in rr.text.split("\n"):
 227                     if task := re.search(rf"<<EJCMC.\s*{CONTEST}\s*,\s*(\w+)", ll):
 228                         TaskDates[task.groups()[0]] = ymd(date)
 229         return jsondump(TaskDates, f)
 230     else:
 231         return json.load(f)
 232 
 233 def runId(U, T):
 234     return Users[U]['Tasks'][T][utRUNID]
 235 
 236 def rwTasks(f, new):
 237     Tasks = {T:{} for T in TaskDates}
 238     if new:
 239         zf = zipfile.ZipFile(f, "w" , zipMETHOD)
 240         for U, A in Members.items():
 241             for T in A['Tasks']:
 242                 runid = A['Tasks'][T][utRUNID]
 243                 try:
 244                     P = Paths[runid]
 245                 except:
 246                     print(f"Error: no {runid} run", file=sys.stderr)
 247                     continue
 248 
 249                 if isfile(fn := join(P, nSOURCE)):
 250                     ft = open(fn)
 251                 elif isfile(fn := join(P, nSOURCE+".gz")):
 252                     ft = gzip.open(fn)
 253                 else:
 254                     print(f"Error: no source in {Runs[runid].Path}", file=sys.stderr)
 255                     continue
 256                 with ft:
 257                     data = ft.read()
 258                     Tasks[T][U] = data if type(data) is str else data.decode()
 259                 # TODO date
 260                 zf.writestr(f"{T}/{U}/{runid}.py", Tasks[T][U])
 261         return zf.close()
 262     else:
 263         zf = zipfile.ZipFile(f, "r")
 264         for entry in zf.infolist():
 265             if entry.is_dir(): continue
 266             T, U, p = entry.filename.split("/")
 267             Tasks[T][U] = zf.read(entry.filename).decode()
 268         return Tasks
 269 
 270 def fixcode(prog):
 271     return autopep8.fix_code(prog, options={'aggressive': 3})
 272 
 273 def rwFormat(f, new):
 274     Format = {T:{} for T in Tasks}
 275     if new:
 276         pool = multiprocessing.Pool()
 277         jobs = [prog for T, users in Tasks.items() for U, prog in users.items()]
 278         # TODO syntax bomb time protection
 279         res = list(reversed(pool.map(fixcode, jobs)))
 280         pool.close()
 281         
 282         for T, users in Tasks.items():
 283             for U, prog in users.items():
 284                 Format[T][U] = res.pop()
 285 
 286         zf = zipfile.ZipFile(f, "w" , zipMETHOD)
 287         for T, users in Format.items():
 288             for U, prog in users.items():
 289                 zf.writestr(f"{T}/{len(prog):05}{sepFORMAT}{int(runId(U, T)):0{lenRun}}{sepFORMAT}{U}.py", Format[T][U])
 290 
 291         return zf.close()
 292     else:
 293         zf = zipfile.ZipFile(f, "r")
 294         for entry in zf.infolist():
 295             if entry.is_dir(): continue
 296             T, prog = entry.filename.split("/")
 297             l, r, U = prog.split(sepFORMAT)
 298             U = U[:-3]
 299             Format[T][U] = zf.read(entry.filename).decode()
 300         return Format
 301 
 302 AstK = ['None ']+[s for s in dir(ast) if s[0].isalpha()]
 303 AstD = {key: chr(char+65) for key, char in zip(AstK, range(len(AstK)))}
 304 reASTBR = re.compile(r"[\[\]\{\}\(\)\,\ ]+")
 305 
 306 def astformat(node):
 307         if isinstance(node, ast.AST):
 308             args = []
 309             for field in node._fields:
 310                     value = getattr(node, field)
 311                     args.append(astformat(value))
 312             return '%s%s' % (AstD[node.__class__.__name__], ''.join(args))
 313         elif isinstance(node, list):
 314             return '%s' % ''.join(astformat(x) for x in node)
 315         return '@'
 316 
 317 def preparate(prog):
 318     return astformat(ast.parse(prog, "p.py"))
 319 
 320 def rwPrep(f, new):
 321     Prep = {}
 322     if new:
 323         pool = multiprocessing.Pool()
 324         jobs = [prog for T, users in Format.items() for U, prog in users.items()]
 325         res = list(reversed(pool.map(preparate, jobs)))
 326         pool.close()
 327 
 328         for T, users in Tasks.items():
 329             for U, prog in users.items():
 330                 Prep.setdefault(T, {})[U] = preparate(prog)
 331         pickle.dump(Prep, f)
 332     else:
 333         return pickle.load(f)
 334 
 335 def smartdist(p1, p2, i, T, u):
 336     if sizeMAXDIFF[0] < len(p1)/len(p2) < sizeMAXDIFF[1] and T not in Dummies:
 337         return distance(p1, p2), i, u
 338     return distMAX, i, u
 339 
 340 def measureClusters():
 341     Clusters = {}
 342     pool = multiprocessing.Pool()
 343     jobs = [prog for T, users in Tasks.items() for U, prog in users.items()]
 344     res = list(reversed(pool.map(autopep8.fix_code, jobs)))
 345     for T, preps in Prep.items():
 346         Clus = [] # clusters: [ [U1, U2, …], [U3, U4, …], …]
 347 
 348         for U, prep in preps.items():
 349             if Clus and U not in fairs(T):
 350                 jobs = [(preps[u], prep, i, T, u) for i,c in enumerate(Clus) for u in c]
 351                 dist, i, u = min(pool.starmap(smartdist, jobs))
 352             else:
 353                 dist, i, u = distMAX, 0, None
 354             if dist >= distMIN:
 355                 Clus.append([U])
 356             else:
 357                 Clus[i].append(U)
 358             Members[U]['Tasks'][T][utDIST:utNEIGH+1] = dist, u
 359             if u and Members[u]['Tasks'][T][utDIST] > dist:
 360                 Members[u]['Tasks'][T][utDIST:utNEIGH+1] = dist, U
 361 
 362         Clusters[T] = Clus
 363     pool.close()
 364     return Clusters, Members
 365 
 366 def rwClusters(f, new):
 367     if new:
 368         return jsondump(measureClusters(), f)
 369     else:
 370         return json.load(f)
 371 
 372 def scoredelay(date, full, gap):
 373     full += int(secDAY*dayBONUS)
 374     return (date <= full)*2 + (date <= full + gap*24*60*60) + 1
 375 
 376 def createUsers(Runs):
 377     global Users
 378     Users = {}
 379     for run in Runs.values():
 380         if run.User_Login not in Users:
 381             Users[run.User_Login] = {
 382                 "UID": int(run.User_Id),
 383                 "Login": run.User_Login,
 384                 "Name": run.User_Name or run.User_Login,
 385                 "Tasks": {},
 386                 "Solved": 0,
 387                 "Score": 0,
 388                 "Penalty": 0,
 389             }
 390         utasks = Users[run.User_Login]["Tasks"]
 391         if run.Prob not in utasks:
 392             # time, fails, deadline status, after penalty, runid, distance, closest-user
 393             utasks[run.Prob] = [0, 0, 0, 0, None, distMAX, None]
 394         if run.Stat_Short == "OK":
 395             if not utasks[run.Prob][utTIME] or utasks[run.Prob][utTIME] > int(run.Time):
 396                 score = scoredelay(1,1,1) if run.Prob in taskSKIP else scoredelay(int(run.Time), TaskDates[run.Prob], dayHALF)
 397                 utasks[run.Prob][utTIME] = int(run.Time)
 398                 utasks[run.Prob][utSTATUS] = utasks[run.Prob][utPENALTY] = score
 399                 utasks[run.Prob][utRUNID] = run.Run_Id
 400         else:
 401             utasks[run.Prob][utFAILS] += 1
 402 
 403 def userPaste(U):
 404     return {T:(res[utNEIGH], res[utDIST]) for T, res in Members[U]['Tasks'].items() if res[utDIST] < distMIN }
 405 
 406 def scoreUsers(Users):
 407     for user, stat in Users.items():
 408         stat["Score"] = 0
 409         # Delete never OK probes
 410         stat["Tasks"] = {k: v for k, v in stat["Tasks"].items() if v[utRUNID] is not None}
 411         for task in stat["Tasks"].values():
 412             stat["Score"] += task[utSTATUS]
 413             if task[2]: # XXX 2 is ut*?
 414                 stat["Solved"] += 1
 415 
 416 def calcformats():
 417     global lenLogin, lenName, lenTask, lenRun
 418     lenLogin = len(max((run.User_Login for run in Runs.values()), key=len))
 419     lenName = len(max((run.User_Name for run in Runs.values()), key=len))
 420     lenTask = len(max(TaskDates, key=len))
 421     lenRun = len(str(len(Runs)))
 422 
 423 def calcscorelimits():
 424     global Maxscore, Minscore, Grades
 425     Maxscore = scoredelay(1,1,1)*len(TaskDates)
 426     Minscore = int(Maxscore * SKIPSCORE)
 427     Grades = dict(reversed([(SCORELINE[i], int(Minscore+(Maxscore-Minscore)*i/len(SCORELINE))) for i in range(len(SCORELINE))]))
 428     Grades['Max'], Grades['Min'] = Maxscore, Minscore
 429 
 430 def ruscore(user, zachot = False):
 431     if user not in Members: return ""
 432     score = Members[user]['Penalty']
 433     if score >= Grades['A']: return "зач" if zachot else "отл"
 434     if score >= Grades['B']: return "зач" if zachot else "хор"
 435     if score >= Grades['C']: return "" if zachot else "удовл"
 436     return ""
 437 
 438 def judgeMembers():
 439     for U in Members:
 440         for T, res in Members[U]['Tasks'].items():
 441             if res[utDIST] < distMIN:
 442                 if not res[utNEIGH]:
 443                     print(f"{U} {T} NO")
 444                 else:
 445                     him = Members[res[utNEIGH]]['Tasks'][T]
 446                     res[utPENALTY] = him[utPENALTY] = 1
 447     for U in Individual:
 448         for T in Individual[U]:
 449             Members[U]['Tasks'][T][utPENALTY] = Individual[U][T]
 450     for U, stat in Members.items():
 451         stat["Penalty"] = 0
 452         for T in stat["Tasks"].values():
 453             stat["Penalty"] += T[utPENALTY]
 454 
 455 
 456 def genMoin(path):
 457     Page, Attach = defaultdict(list), []
 458     Page[""].append(f"= {CONTEST} results =")
 459     Page[""].append(f"|| '''User''' || '''Name ''' || '''Grade''' || '''Pass''' || '''Score/Max''' ||") 
 460     for U, A in sorted(Users.items()):
 461         if U in Members: A = Members[U]
 462         N, oc, za, P, S = A['Name'], ruscore(U), ruscore(U, 1), A['Penalty'], A['Score']
 463         Page[""].append(f"|| [[#{U}|{U}]] || <<Anchor(_{U})>>{N} || {oc}|| {za} || {P}/{S} ||") 
 464         Page[U].append(f"=== {A['Name']} ===")
 465         Page[U].append(f"<<Anchor({U})>>[[#_{U}|Back]]")
 466         for T, (time, fails, actual, score, runid, dist, neigh) in A['Tasks'].items():
 467             if dist < distMIN:
 468                 aname = f"diff_{T}_{U}_{neigh}.html"
 469                 Attach.append((aname, Format[T][U], Format[T][neigh], T+"/"+U, T+"/"+neigh))
 470                 diff = f"[[{path}/{aname}|{neigh} = {dist}]]"
 471             else:
 472                 diff = ""
 473             time, day = date(time), date(TaskDates[T])
 474             Page[U].append(f"|| `{T}` || '''{score}'''/{actual} || {time}/{day} || {runid} || {diff} ||")
 475     repdir = join(dCACHE, nREPORT)
 476     if(os.path.isdir(repdir)):
 477         shutil.rmtree(repdir)
 478     os.makedirs(join(repdir, nATTACHMENTS))
 479     fcontent = join(repdir, "content.moin")
 480     with open(fcontent, "w") as f:
 481         f.writelines([s+'\n' for t in Page.values() for s in t])
 482     for att, t1, t2, u1, u2 in Attach:
 483         diff = dohtmldiff(t1, t2, u1, u2)
 484         with open(join(repdir, nATTACHMENTS, att), "w") as f:
 485             f.write(diff)
 486     print(fcontent)
 487 
 488 def init():
 489     global Runs, Paths, TaskDates, Members, Tasks, Format, Prep, Clusters
 490     t = time.time()
 491     os.makedirs(dCACHE, exist_ok=True)
 492     Paths = Cached(nPATHS, rwPaths)
 493     Runs, TaskDates = processData(*Cached(nLOAD, rwDownload))
 494     createUsers(Runs)
 495     calcformats()
 496     calcscorelimits()
 497     scoreUsers(Users)
 498     Members = {k: v for k, v in Users.items() if v['Score'] >= Minscore }
 499     Tasks = Cached(nTASKS, rwTasks, "b")
 500     Format = Cached(nFORMAT, rwFormat, "b")
 501     Prep = Cached(nPREP, rwPrep, "b")
 502     Clusters, Members = Cached(nDISTS, rwClusters)
 503     judgeMembers()
 504     # TODO Final score HTML tree
 505     print(f"Time elapsed: {time.time()-t:.1f}")
 506 
 507 def erase(*Targets, Full=False, Empty=False):
 508     for entry in iglob(join(dCACHE,"*"), recursive=False):
 509         if isfile(entry) and (not Empty or os.stat(entry).st_size==0) \
 510                          and (Full or basename(entry) not in namesKEEP) \
 511                          and (not Targets or basename(entry) in Targets):
 512             print(f"Erasing {basename(entry)}")
 513             os.unlink(entry)
 514 
 515 class Shell(cmd.Cmd):
 516     intro = f"\n\t{CONTEST} contest +{dayBONUS} day bonus deadline\n"
 517     prompt = "Python2020> "
 518     curuser, curtask = None, None
 519 
 520     def do_users(self, arg):
 521         '''
 522         users       List actual users
 523         users MIN   List users who reach at list MIN score
 524                     MIN is A, B, C, Min, Max or number
 525         users ALL   List all users
 526         '''
 527         Staff = Users if arg in ("ALL", "all") else Members
 528         Total, Min = 0, Grades.get(arg, int(arg) if arg.isdigit() else 0)
 529         for S in sorted(Staff.values(), key=lambda u: u['Penalty']):
 530             if S['Penalty'] >= Min:
 531                 Total += 1 
 532                 print(f"{S['Login']:{lenLogin}}: {ruscore(S['Login'])}/{ruscore(S['Login'], 1)}: {S['Penalty']:3}/{S['Score']} {S['Name']}")
 533         print(f"{Total=}")
 534 
 535     def complete_users(self, text, line, begidx, endidx):
 536         return [grade for grade in ['ALL']+Grades if grade.startswith(text)]
 537     
 538     def do_user(self, arg):
 539         '''
 540         user USER           Show any USER statistics
 541         user USER TASK N    Set individual USER/TASK score to N
 542         user USER TASK -    Unset individual USER/TASK score
 543         '''
 544 
 545         if len(arg.split()) == 3:
 546             u, t, n = arg.split()
 547             if u in Members and t in Tasks:
 548                 if n.isdigit():
 549                     Individual.setdefault(u, {})[t] = int(n)
 550                     saveconfig()
 551                 elif n.startswith('-'):
 552                     if u in Individual and t in Individual[u]:
 553                         del Individual[u][t]
 554                     saveconfig()
 555                 else:
 556                     print(f"Invalid score {n}")
 557             else:
 558                 print(f"Cannot set {u} score for {t}")
 559             return
 560 
 561         arg = self.curuser = arg if arg in Users else self.curuser or list(Users)[0]
 562         U = Members[arg] if arg in Members else Users[arg]
 563         print(f"\t{U['Name']} ({U['Login']}):")
 564         for T in sorted(U['Tasks'], key=lambda t: U['Tasks'][t][utPENALTY], reverse=True):
 565             time, fails, actual, score, runid, dist, neigh = U['Tasks'][T]
 566             t1, t2, dt = date(TaskDates[T]), date(time), int((time-TaskDates[T])/60/60/24)
 567             print(f"{T:<{lenTask}}: {score}/{actual} ({dt} = {t1} - {t2}) {fails=:<2}")
 568         print(f"{arg in Members and 'PASSED' or 'FAILED'}: Total {U['Solved']}/{len(TaskDates)}, score: {U['Penalty']}/{Maxscore}")
 569         if arg in Individual:
 570             print("Individual scores:")
 571             for t in Individual[arg]:
 572                 print("\t", t, Individual[arg][t])
 573         if arg in Members:
 574             if pastes := userPaste(arg):
 575                 print()
 576                 for T, (u, d) in pastes.items():
 577                     print(f"{T}: {u}, distance {d}")
 578             print(f"\t{ruscore(arg)} / {ruscore(arg, 1)}")
 579 
 580     def complete_user(self, text, line, begidx, endidx):
 581         cmd = line[:begidx].split()
 582         if len(cmd) == 1:
 583             return [name for name in Users if name.startswith(text)]
 584         if len(cmd) == 2:
 585             return [name for name in Tasks if name.startswith(text)]
 586         if len(cmd) == 3:
 587             return list('01234')
 588 
 589     def do_compare(self, arg):
 590         '''
 591         compare USER1 USER2     Compare and show distances of any solution
 592         '''
 593         if len(arg.split()) != 2: return
 594         U, H = arg.split()
 595         for u in U, H:
 596             if u not in Members:
 597                 print(f"Unknown user '{u}'")
 598                 return
 599 
 600         D = sorted((distance(Prep[T][U], Prep[T][H]), T) for T in Tasks if U in Prep[T] and H in Prep[T])
 601         print("{U} {H} distances:", *reversed(D), sep="\n")
 602 
 603     def complete_compare(self, text, line, begidx, endidx):
 604         return [name for name in Members if name.startswith(text)]
 605 
 606     def do_manualscores(self, arg):
 607         '''
 608         List individual scores. Use "user task score" to modify
 609         '''
 610         for U in Individual:
 611             print(f"{U}:", end=" ")
 612             for T, s in Individual[U].items():
 613                 print(f"{T} {s};", end=" ")
 614             print()
 615 
 616     def do_info(self, arg):
 617         '''
 618         Show overaLL info
 619         '''
 620         Actual = sum(len(T['Tasks']) for T in Members.values())
 621         print(f"{CONTEST=} Tasks: {len(TaskDates)} Tries: {len(Runs)}")
 622         print(f"{Grades=}")
 623         print(f"Users: {len(Users)}/{len(Members)} (tasks in question: {Actual})")
 624 
 625     def do_reset(self, arg):
 626         '''
 627         Re-create databases
 628         '''
 629         erase(*arg.split())
 630         init()
 631 
 632     def complete_reset(self, text, line, begidx, endidx):
 633         return [name for name in namesRESET if name.startswith(text)]
 634     
 635     def do_set(self, arg):
 636         '''
 637         set VAR=[VALUE]     Display ejstat.py global namespace object VAR od set it to VALUE
 638         '''
 639         name, *val = arg.split("=")
 640         if val:
 641             globals()[name] = eval(val[0])
 642         elif name:
 643             print(f"{name}={globals()[name]}")
 644         else:
 645             for name, val in globals().items():
 646                 print(f"{name}={repr(val)[:60]}")
 647 
 648     def complete_set(self, text, line, begidx, endidx):
 649         return [name for name in globals() if name.startswith(text) and not name.startswith("_") and not callable(globals()[name])]
 650     
 651     def do_tasks(self, arg):
 652         '''
 653         Show all tasks along with copypaste clusters
 654         '''
 655         for T, Clus in Clusters.items():
 656             print(f"\n\t{T}:")
 657             if max(map(len, Clus)) >= sizeMAXCLUST:
 658                 res = "COMMON"
 659             else:
 660                 res = "\n".join(f"{len(C)}: "+" ".join(C) for C in Clus if len(C)>1)
 661             if res: print(res)
 662 
 663     def do_task(self, arg):
 664         '''
 665         task [TASK]            Show TASK statistics
 666         task TASK USER         Show USER solution of TASK
 667         task TASK USER1 USER1  Show diffference beween USER1 ans USER2 solutions
 668         '''
 669         if not arg:
 670             arg = self.curtask or list(TaskDates)[-1]
 671         cmd = arg.split()
 672         if cmd[0] not in Tasks: return print(f"No {cmd[0]} task")
 673         self.curtask = cmd[0]
 674         print(f"\n\t{self.curtask}:")
 675         if len(cmd) == 1:
 676             print("1:", *(C[0] for C in Clusters[cmd[0]] if len(C) == 1))
 677             print("\n".join(f"{len(C)}: "+" ".join(C) for C in Clusters[cmd[0]] if len(C)>1))
 678             return
 679         users = Format[cmd[0]]
 680         if cmd[1] not in users: return print(f"No {cmd[1]} user")
 681         elif len(cmd) == 2:
 682             print(highlight(users[cmd[1]], "Python3Lexer", "Terminal256Formatter"))
 683             return
 684         if cmd[2] not in users: return print(f"No {cmd[2]} user")
 685         elif len(cmd) == 3:
 686             r1, r2 = runId(cmd[1], cmd[0]), runId(cmd[2], cmd[0])
 687             if int(r1) > int(r2):
 688                 r1, r2, cmd[1], cmd[2] = r2, r1, cmd[2], cmd[1]
 689             res = dodiff(users[cmd[1]], users[cmd[2]], f"{r1}-{cmd[1]}", f"{r2}-{cmd[2]}")
 690             dist = distance(Prep[cmd[0]][cmd[1]], Prep[cmd[0]][cmd[2]])
 691             if res:
 692                 print(highlight("\n".join(res), "DiffLexer", "TerminalFormatter"))
 693             if res and distance:
 694                 print(f"Distance: {dist}")
 695             else:
 696                 print("!!! IDENTICAL !!!")
 697         elif len(cmd) >= 4:
 698             dfile = join(dCACHE, f"{nDIFFHTML}-{cmd[1]}-{cmd[2]}.html")
 699             diff = dohtmldiff(users[cmd[1]], users[cmd[2]], cmd[1], cmd[2])
 700             with open(dfile, "w") as f:
 701                 f.write(diff)
 702             print(f"{dfile}")
 703             if len(cmd)> 4:
 704                 subprocess.Popen(["firefox", dfile])
 705 
 706     def complete_task(self, text, line, begidx, endidx):
 707         pos = len(cmd := line[:begidx].split())
 708         if pos == 1:
 709             return [name for name in Tasks if name.startswith(text)]
 710         if pos in (2, 3):
 711             return [name for name in Format[cmd[1]] if name.startswith(text)]
 712         if pos >= 4:
 713             return ["html"]
 714 
 715     def do_report(self, arg):
 716         '''
 717         report PATH     Generate Moin-style text report and diff files in PATH
 718         '''
 719         genMoin(arg)
 720 
 721     def do_who(self, arg):
 722         '''
 723         who TEXT        Search users with name TEXT
 724         '''
 725         parts = arg.lower().split()
 726         for U, A in Users.items():
 727             if all(s in A['Name'].lower() for s in parts) or any(s in U for s in parts):
 728                 print(f"{U}: {A['Name']}")
 729 
 730     def do_config(self, arg):
 731         '''
 732         config                  Print current configuration
 733         config dummy [[-]TASK]  Print dummy tasks or add/remove TASK from dummies
 734         config fair [[-]TASK [USER]]
 735                                 Print fair clusters or update them
 736         '''
 737         global Dummies
 738         cmd = arg.split()
 739         if len(cmd) < 2:
 740             if not cmd or cmd[0]=="dummy":
 741                 print("Dummies:", *Dummies)
 742             if not cmd or cmd[0]=="fair":
 743                 print("Fair clusters:")
 744                 pprint.pprint(Fairset)
 745             if not cmd or cmd[0]=="individual":
 746                 print("Individual scores:")
 747                 pprint.pprint(Individual)
 748             return
 749         if remove := cmd[1].startswith("-"):
 750             cmd[1] = cmd[1][1:]
 751         if cmd[1] not in Tasks:
 752             print(f"Unknown task {cmd[1]}")
 753         elif len(cmd) >= 2 and cmd[0] == "dummy":
 754             if remove:
 755                 Dummies -= {cmd[1]}
 756             else:
 757                 Dummies.add(cmd[1])
 758             saveconfig()
 759         elif cmd[0] == "fair":
 760             if len(cmd) == 2:
 761                 print(f"Fair in {cmd[1]}:", *fairs(cmd[1]))
 762             else:
 763                 if cmd[2] not in Users:
 764                     print(f"Unknown user {cmd[2]}")
 765                 else:
 766                     if remove:
 767                         if fa := fairs(cmd[1]) and cmd[2] in fa:
 768                             fa.remove(cmd[2])
 769                     else:
 770                         fairs(cmd[1], cmd[2])
 771                 saveconfig()
 772         else:
 773             print(f"Unknown command {arg}")
 774 
 775     def complete_config(self, text, line, begidx, endidx):
 776         cmd = line[:begidx].split()
 777         if len(cmd) == 1:
 778             return [name for name in ("dummy", "fair") if name.startswith(text)]
 779         elif len(cmd) == 2:
 780             return [name for name in Tasks if name.startswith(text)]
 781         elif cmd[1] == 'fair':
 782             return [name for name in Members if name.startswith(text)]
 783 
 784     def do_eval(self, arg):
 785         '''
 786         eval EXPR       Print arbitrary eval(EXPR) on ejstat.py global namespace
 787         '''
 788         print(eval(arg))
 789 
 790     def complete_eval(self, text, line, begidx, endidx):
 791         return [name for name in globals() if name.startswith(text)]
 792 
 793     def preloop(self):
 794         global HFile
 795         if not HFile:
 796             import readline
 797             HFile = join(dCACHE, nHISTORY)
 798             if isfile(HFile):
 799                 readline.read_history_file(HFile)
 800                 readline.set_history_length(1000)
 801             atexit.register(readline.write_history_file, HFile)
 802 
 803     def emptyline(self):
 804         pass
 805 
 806     def do_EOF(self, arg):
 807         return True
 808 
 809 def main():
 810     erase(Full="!!!" in sys.argv, Empty="!" not in sys.argv)
 811     init()
 812     while True:
 813         try:
 814             Shell().cmdloop()
 815         except Exception as E:
 816             traceback.print_tb(E.__traceback__)
 817             print(f"{type(E).__name__}: {E}")
 818         except KeyboardInterrupt as E:
 819             print("^C")
 820         else:
 821             break
 822 
 823 main()