From 63ab94faef3c1b143c577721dee2522dbe6bda7a Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Sun, 22 Dec 2024 03:06:28 +0300 Subject: [PATCH 1/6] intro query --- elixir/query.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/elixir/query.py b/elixir/query.py index 1e740bf2..2abbdad2 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -331,4 +331,31 @@ def get_idents_defs(self, version, ident, family): symbol_doccomments.append(SymbolInstance(path, docline)) return symbol_definitions, symbol_references, symbol_doccomments + + def get_peeks_of_syms(self, version, symbol_definitions, symbol_references): + + peeks = {} + + def request_peeks(syms): + if len(syms) > 100: + return + for sym in syms: + if sym.path not in peeks: + peeks[sym.path] = {} + + content = self.scriptLines('get-file', version, "/" + sym.path) + + if type(sym.line) is int: + lines = (sym.line,) + else: + lines = map(int, sym.line.split(',')) + + for num in lines: + index = num - 1 + if index >= 0 and index < len(content): + peeks[sym.path][num] = decode(content[index]).strip() + + request_peeks(symbol_definitions) + request_peeks(symbol_references) + return peeks From 86a63d1bc953a8bc86c9ecfc2914fa69e8b9e1f3 Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Sun, 22 Dec 2024 03:11:11 +0300 Subject: [PATCH 2/6] use peeks in dynamic ref --- elixir/api.py | 3 ++- static/dynamic-references.js | 31 +++++++++++++++++++++++-------- static/style.css | 11 +++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/elixir/api.py b/elixir/api.py index 57936f3c..730a0c97 100755 --- a/elixir/api.py +++ b/elixir/api.py @@ -53,7 +53,8 @@ def on_get(self, req, resp, project, ident): resp.media = { 'definitions': [sym.__dict__ for sym in symbol_definitions], 'references': [sym.__dict__ for sym in symbol_references], - 'documentations': [sym.__dict__ for sym in symbol_doccomments] + 'documentations': [sym.__dict__ for sym in symbol_doccomments], + 'peeks': peaks } query.close() diff --git a/static/dynamic-references.js b/static/dynamic-references.js index c2816bac..4ea94464 100644 --- a/static/dynamic-references.js +++ b/static/dynamic-references.js @@ -28,7 +28,7 @@ function identUrl(project, ident, version, family) { ] */ -function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) { +function generateSymbolDefinitionsHTML(symbolDefinitions, peeks, project, version) { let result = ""; let typesCount = {}; let previous_type = ""; @@ -55,7 +55,7 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) { previous_type = sd.type; } let ln = sd.line.toString().split(','); - if (ln.length == 1) { + if (ln.length == 1 && !peeks) { let n = ln[0]; result += `
  • ${sd.path}, line ${n} (as a ${sd.type})`; } else { @@ -66,7 +66,14 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) { result += `
  • ${sd.path} (as a ${sd.type})`; result += ''; } @@ -77,7 +84,7 @@ function generateSymbolDefinitionsHTML(symbolDefinitions, project, version) { return result; } -function generateSymbolReferencesHTML(symbolReferences, project, version) { +function generateSymbolReferencesHTML(symbolReferences, peeks, project, version) { let result = ""; if(symbolReferences.length == 0) { @@ -88,7 +95,7 @@ function generateSymbolReferencesHTML(symbolReferences, project, version) { result += '
      '; for (let sr of symbolReferences) { let ln = sr.line.split(','); - if (ln.length == 1) { + if (ln.length == 1 && !peeks) { let n = ln[0]; result += `
    • ${sr.path}, line ${n}`; } else { @@ -99,7 +106,14 @@ function generateSymbolReferencesHTML(symbolReferences, project, version) { result += `
    • ${sr.path}`; result += '' } @@ -143,10 +157,11 @@ function generateReferencesHTML(data, project, version) { let symbolDefinitions = data["definitions"]; let symbolReferences = data["references"]; let symbolDocumentations = data["documentations"]; + let peeks = data["peeks"]; return '
      ' + generateDocCommentsHTML(symbolDocumentations, project, version) + - generateSymbolDefinitionsHTML(symbolDefinitions, project, version) + - generateSymbolReferencesHTML(symbolReferences, project, version) + + generateSymbolDefinitionsHTML(symbolDefinitions, peeks, project, version) + + generateSymbolReferencesHTML(symbolReferences, peeks, project, version) + '
      '; } diff --git a/static/style.css b/static/style.css index 77d7703c..1d364c51 100644 --- a/static/style.css +++ b/static/style.css @@ -718,6 +718,12 @@ h2 { font-weight: 400; color: #000; } + +.lxrident a span { + width: 8em; + display: inline-block; +} + .lxrident li li::before { color: #444; } @@ -739,6 +745,11 @@ h2 { content: '└╴'; } +.lxrident li pre { + display: inline-block; + margin: auto; +} + /* tree */ From 990b41bdbfce4ca0b2de766ad64fd8532ffbfab7 Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Sun, 22 Dec 2024 03:12:34 +0300 Subject: [PATCH 3/6] use peeks in jinja --- elixir/web.py | 20 ++++++++++++-------- templates/ident.html | 10 +++++++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/elixir/web.py b/elixir/web.py index 23b1265c..336a6b5c 100755 --- a/elixir/web.py +++ b/elixir/web.py @@ -662,11 +662,12 @@ def generate_source_page(ctx: RequestContext, q: Query, # type : type of the symbol # path: path of the file that contains the symbol # line: list of LineWithURL -SymbolEntry = namedtuple('SymbolEntry', 'type, path, lines') +# peeks: map of code line previews for this path +SymbolEntry = namedtuple('SymbolEntry', 'type, path, lines, peeks') # Converts SymbolInstance into SymbolEntry # path of SymbolInstance will be appended to base_url -def symbol_instance_to_entry(base_url: str, symbol: SymbolInstance) -> SymbolEntry: +def symbol_instance_to_entry(base_url, symbol, peeks): # TODO this should be a responsibility of Query if type(symbol.line) is str: line_numbers = symbol.line.split(',') @@ -674,11 +675,13 @@ def symbol_instance_to_entry(base_url: str, symbol: SymbolInstance) -> SymbolEnt line_numbers = [symbol.line] lines = [ - LineWithURL(l, f'{ base_url }/{ symbol.path }#L{ l }') + LineWithURL(int(l), f'{ base_url }/{ symbol.path }#L{ l }') for l in line_numbers ] - return SymbolEntry(symbol.type, symbol.path, lines) + current_peeks = peeks.get(symbol.path, {}) + + return SymbolEntry(symbol.type, symbol.path, lines, current_peeks) # Generates response (status code and optionally HTML) of the `ident` route # basedir: path to data directory, ex: "/srv/elixir-data" @@ -689,12 +692,13 @@ def generate_ident_page(ctx: RequestContext, q: Query, source_base_url = get_source_base_url(project, version) symbol_definitions, symbol_references, symbol_doccomments = q.search_ident(version, ident, family) symbol_sections = [] + empty_peeks = {} if len(symbol_definitions) or len(symbol_references): if len(symbol_doccomments): symbol_sections.append({ 'title': 'Documented', - 'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym) for sym in symbol_doccomments]}, + 'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym, empty_peeks) for sym in symbol_doccomments]}, }) if len(symbol_definitions): @@ -703,9 +707,9 @@ def generate_ident_page(ctx: RequestContext, q: Query, # TODO this should be a responsibility of Query for sym in symbol_definitions: if sym.type not in defs_by_type: - defs_by_type[sym.type] = [symbol_instance_to_entry(source_base_url, sym)] + defs_by_type[sym.type] = [symbol_instance_to_entry(source_base_url, sym, peeks)] else: - defs_by_type[sym.type].append(symbol_instance_to_entry(source_base_url, sym)) + defs_by_type[sym.type].append(symbol_instance_to_entry(source_base_url, sym, peeks)) symbol_sections.append({ 'title': 'Defined', @@ -719,7 +723,7 @@ def generate_ident_page(ctx: RequestContext, q: Query, if len(symbol_references): symbol_sections.append({ 'title': 'Referenced', - 'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym) for sym in symbol_references]}, + 'symbols': {'_unknown': [symbol_instance_to_entry(source_base_url, sym, peeks) for sym in symbol_references]}, }) else: symbol_sections.append({ diff --git a/templates/ident.html b/templates/ident.html index 7781494d..67f426df 100644 --- a/templates/ident.html +++ b/templates/ident.html @@ -33,7 +33,7 @@

      {{ section['title'] }} in {{ symbols|length }} files
    • {{ symbol.path }} - {%- if symbol.lines|length == 1 %}, + {%- if symbol.lines|length == 1 and symbol.peeks|length == 0 %}, line {{ symbol.lines[0].lineno }} {%- elif symbols|length > 100 %}, {{ symbol.lines|length }} times @@ -42,10 +42,14 @@

      {{ section['title'] }} in {{ symbols|length }} files (as a {{ symbol.type }}) {% endif %} - {% if symbol.lines|length > 1 and symbols|length <= 100 %} + {% if (symbol.lines|length > 1 or symbol.peeks|length > 0) and symbols|length <= 100 %} {% endif %} From b00bbe0f463984c8f11120adc1a5bb08e0e20d64 Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Sat, 15 Feb 2025 18:05:22 +0300 Subject: [PATCH 4/6] use batch --- elixir/lib.py | 6 +++--- elixir/query.py | 33 +++++++++++++++++++++++++++------ script.sh | 9 +++++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/elixir/lib.py b/elixir/lib.py index 7d7d0757..655de093 100755 --- a/elixir/lib.py +++ b/elixir/lib.py @@ -26,15 +26,15 @@ CURRENT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../') -def script(*args, env=None): +def script(*args, input=None, env=None): args = (os.path.join(CURRENT_DIR, 'script.sh'),) + args # subprocess.run was introduced in Python 3.5 # fall back to subprocess.check_output if it's not available if hasattr(subprocess, 'run'): - p = subprocess.run(args, stdout=subprocess.PIPE, env=env) + p = subprocess.run(args, stdout=subprocess.PIPE, input=input, env=env) p = p.stdout else: - p = subprocess.check_output(args) + p = subprocess.check_output(args, input=input) return p def run_cmd(*args, env=None): diff --git a/elixir/query.py b/elixir/query.py index 2abbdad2..ec70bf72 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -63,8 +63,8 @@ def __init__(self, data_dir, repo_dir): self.db = data.DB(data_dir, readonly=True, dtscomp=self.dts_comp_support) self.file_cache = {} - def script(self, *args): - return script(*args, env=self.getEnv()) + def script(self, *args, input=None): + return script(*args, input=input, env=self.getEnv()) def scriptLines(self, *args): return scriptLines(*args, env=self.getEnv()) @@ -331,7 +331,29 @@ def get_idents_defs(self, version, ident, family): symbol_doccomments.append(SymbolInstance(path, docline)) return symbol_definitions, symbol_references, symbol_doccomments - + + def get_files_and_zip(self, version, syms): + batch = b"\n".join([f"{version}:{sym.path}".encode() for sym in syms]) + batch_res = self.script('get-files-batch', input=batch) + + # See https://git-scm.com/docs/git-cat-file#_batch_output for the format: + # + # SP SP LF + # LF + # SP SP LF + # LF + # SP SP LF + # LF + # + for sym in syms: + meta, batch_res = batch_res.split(b"\n", 1) + _, _, size = meta.split(b" ") + size = int(size) + 1 # newline after each file + content = batch_res[:size].split(b"\n") + batch_res = batch_res[size:] + yield sym, content + + def get_peeks_of_syms(self, version, symbol_definitions, symbol_references): peeks = {} @@ -339,12 +361,11 @@ def get_peeks_of_syms(self, version, symbol_definitions, symbol_references): def request_peeks(syms): if len(syms) > 100: return - for sym in syms: + + for sym, content in self.get_files_and_zip(version, syms): if sym.path not in peeks: peeks[sym.path] = {} - content = self.scriptLines('get-file', version, "/" + sym.path) - if type(sym.line) is int: lines = (sym.line,) else: diff --git a/script.sh b/script.sh index 3bbff2a7..c5b9fe69 100755 --- a/script.sh +++ b/script.sh @@ -85,6 +85,11 @@ get_file() git cat-file blob "$v:`denormalize $opt2`" 2>/dev/null } +get_files_batch() +{ + git cat-file --batch 2>/dev/null +} + get_dir() { v=`echo $opt1 | version_rev` @@ -263,6 +268,10 @@ case $cmd in get_file ;; + get-files-batch) + get_files_batch + ;; + get-dir) get_dir ;; From 3082ac2833e5eb5fe3ba5085b70a9bc234807819 Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Sat, 15 Feb 2025 23:30:40 +0300 Subject: [PATCH 5/6] fix review --- elixir/api.py | 2 +- elixir/query.py | 2 -- elixir/web.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/elixir/api.py b/elixir/api.py index 730a0c97..c9232352 100755 --- a/elixir/api.py +++ b/elixir/api.py @@ -54,7 +54,7 @@ def on_get(self, req, resp, project, ident): 'definitions': [sym.__dict__ for sym in symbol_definitions], 'references': [sym.__dict__ for sym in symbol_references], 'documentations': [sym.__dict__ for sym in symbol_doccomments], - 'peeks': peaks + 'peeks': peeks } query.close() diff --git a/elixir/query.py b/elixir/query.py index ec70bf72..71d0d453 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -353,9 +353,7 @@ def get_files_and_zip(self, version, syms): batch_res = batch_res[size:] yield sym, content - def get_peeks_of_syms(self, version, symbol_definitions, symbol_references): - peeks = {} def request_peeks(syms): diff --git a/elixir/web.py b/elixir/web.py index 336a6b5c..582bf49e 100755 --- a/elixir/web.py +++ b/elixir/web.py @@ -27,7 +27,7 @@ import datetime from collections import OrderedDict, namedtuple from re import search, sub -from typing import Any, Callable, NamedTuple, Tuple +from typing import Any, Callable, Dict, NamedTuple, Tuple from urllib import parse import falcon import jinja2 @@ -667,7 +667,7 @@ def generate_source_page(ctx: RequestContext, q: Query, # Converts SymbolInstance into SymbolEntry # path of SymbolInstance will be appended to base_url -def symbol_instance_to_entry(base_url, symbol, peeks): +def symbol_instance_to_entry(base_url: str, symbol: SymbolInstance, peeks: Dict[str, Dict[int, str]]) -> SymbolEntry: # TODO this should be a responsibility of Query if type(symbol.line) is str: line_numbers = symbol.line.split(',') From d8c7c1006d427b00685de224296a6347555d1c97 Mon Sep 17 00:00:00 2001 From: Daniil159x Date: Mon, 14 Jul 2025 00:59:20 +0300 Subject: [PATCH 6/6] fix rebase --- elixir/api.py | 2 +- elixir/query.py | 7 +++++-- elixir/web.py | 2 +- utils/query.py | 7 ++++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/elixir/api.py b/elixir/api.py index c9232352..5674f984 100755 --- a/elixir/api.py +++ b/elixir/api.py @@ -46,7 +46,7 @@ def on_get(self, req, resp, project, ident): if version == 'latest': version = query.get_latest_tag() - symbol_definitions, symbol_references, symbol_doccomments = query.search_ident(version, ident, family) + symbol_definitions, symbol_references, symbol_doccomments, peeks = query.search_ident(version, ident, family) resp.status = falcon.HTTP_200 resp.content_type = falcon.MEDIA_JSON diff --git a/elixir/query.py b/elixir/query.py index 71d0d453..8966b51e 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -179,9 +179,12 @@ def get_file_type(self, version, path): def search_ident(self, version, ident, family): # DT bindings compatible strings are handled differently if family == 'B': - return self.get_idents_comps(version, ident) + defs, refs, docs = self.get_idents_comps(version, ident) else: - return self.get_idents_defs(version, ident, family) + defs, refs, docs = self.get_idents_defs(version, ident, family) + + peeks = self.get_peeks_of_syms(version, defs, refs) + return defs, refs, docs, peeks # Returns the latest tag that is included in the database. # This excludes release candidates. diff --git a/elixir/web.py b/elixir/web.py index 582bf49e..73ff2e68 100755 --- a/elixir/web.py +++ b/elixir/web.py @@ -690,7 +690,7 @@ def generate_ident_page(ctx: RequestContext, q: Query, status = falcon.HTTP_OK source_base_url = get_source_base_url(project, version) - symbol_definitions, symbol_references, symbol_doccomments = q.search_ident(version, ident, family) + symbol_definitions, symbol_references, symbol_doccomments, peeks = q.search_ident(version, ident, family) symbol_sections = [] empty_peeks = {} diff --git a/utils/query.py b/utils/query.py index fbd35af5..e7103a80 100644 --- a/utils/query.py +++ b/utils/query.py @@ -16,7 +16,7 @@ def cmd_versions(q, **kwargs): print(v) def cmd_ident(q, version, ident, family, **kwargs): - symbol_definitions, symbol_references, symbol_doccomments = q.search_ident(version, ident, family) + symbol_definitions, symbol_references, symbol_doccomments, peeks = q.search_ident(version, ident, family) print("Symbol Definitions:") for symbol_definition in symbol_definitions: print(symbol_definition) @@ -29,6 +29,11 @@ def cmd_ident(q, version, ident, family, **kwargs): for symbol_doccomment in symbol_doccomments: print(symbol_doccomment) + print("\nSymbol peeks:") + for file, content in peeks.items(): + for num, line in content.items(): + print(f"{file}:{num}: {line}") + def cmd_file(q, version, path, **kwargs): code = q.get_tokenized_file(version, path) print(code)