Refactor EurostatData to improve dataset handling and add theme selection functionality (#1005)

* Refactor EurostatData to improve dataset handling and add theme selection functionality

* Show download progress

* Refactor EurostatData to streamline theme and subtheme selection, enhancing dataset handling and user interaction

* Add initial meta data processing scripts and configuration files

* Fix test

* Optimize JSONStreamParser to improve performance by adjusting the frequency of yielding control during parsing based on time and count thresholds.

* Improve streaming and progress bar

Author: ishii-norimi

js/data/eurostat.js

@@ -0,0 +1 @@
+node_modules

js/data/meta/.gitignore

@@ -0,0 +1,6 @@
+{
+  "type": "module",
+  "devDependencies": {
+    "fast-xml-parser": "^5.2.1"
+  }
+}

js/data/meta/catalogue.json.gz

@@ -0,0 +1,120 @@
+import fs from 'fs/promises'
+import path from 'path'
+import { XMLParser } from 'fast-xml-parser'
+
+const basePath = import.meta.dirname
+
+const fetchProgress = async (input, init) => {
+	let waitTime = 0.01
+	const units = ['B', 'KiB', 'MiB', 'GiB']
+	for (let i = 0; i < 100; i++) {
+		try {
+			const response = await fetch(input, init)
+			const total = response.headers.get('content-length')
+			let loaded = 0
+			let prevPrintTime = 0
+
+			const progressdStream = new TransformStream({
+				transform(chunk, controller) {
+					controller.enqueue(chunk)
+					loaded += chunk.byteLength
+					const now = Date.now()
+					if (now - prevPrintTime > 1000) {
+						prevPrintTime = now
+						if (total) {
+							console.info(loaded + ' / ' + total + ' bytes')
+						} else {
+							let uidx = 0
+							let val = loaded
+							while (val > 1000 && uidx + 1 < units.length) {
+								val /= 1024
+								uidx++
+							}
+							console.info(Math.floor(val * 100) / 100 + ' ' + units[uidx])
+						}
+					}
+				},
+			})
+
+			return new Response(response.body.pipeThrough(progressdStream))
+		} catch {
+			await new Promise(resolve => setTimeout(resolve, waitTime * 1000))
+			waitTime = Math.min(waitTime * 2, 5)
+		}
+	}
+	return null
+}
+
+const getCatalogue = async () => {
+	console.log('Fetching catalogue...')
+	const res = await fetchProgress('https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/xml')
+	if (!res) {
+		throw new Error('Failed to fetch catalogue')
+	}
+	const catalogues = await res.text()
+	const parser = new XMLParser({ ignoreAttributes: false })
+	const doc = parser.parse(catalogues, 'application/xml')
+	const themes = doc['nt:tree']['nt:branch'][0]
+
+	const getLeafs = node => {
+		const obj = {
+			title: node['nt:title'][0]['#text'],
+			children: [],
+		}
+		const branch = node['nt:children']['nt:branch']
+		if (branch) {
+			for (const b of Array.isArray(branch) ? branch : [branch]) {
+				obj.children.push(getLeafs(b))
+			}
+		} else {
+			const leafs = node['nt:children']['nt:leaf']
+			for (const leaf of Array.isArray(leafs) ? leafs : [leafs]) {
+				obj.children.push({
+					title: leaf['nt:title'][0]['#text'],
+					code: leaf['nt:code'],
+				})
+			}
+		}
+		return obj
+	}
+
+	const root = getLeafs(themes)
+
+	const readableStream = new Blob([JSON.stringify(root)]).stream()
+	const compressedStream = readableStream.pipeThrough(new CompressionStream('gzip'))
+	const arrayBuffer = await new Response(compressedStream).arrayBuffer()
+	fs.writeFile(path.join(basePath, 'catalogue.json.gz'), Buffer.from(arrayBuffer))
+}
+
+const getMetabase = async () => {
+	console.log('Fetching metabase...')
+	let res = await fetchProgress('https://ec.europa.eu/eurostat/api/dissemination/catalogue/metabase.txt.gz')
+	if (!res) {
+		throw new Error('Failed to fetch metadata')
+	}
+	const decompressedStream = res.body.pipeThrough(new DecompressionStream('gzip'))
+	const text = await new Response(decompressedStream).text()
+
+	const data = {}
+	for (const line of text.split('\n')) {
+		if (line.trim().length === 0) {
+			continue
+		}
+		const [id, unit, value] = line.split('\t')
+		if (!data[id]) {
+			data[id] = {}
+		}
+		if (!data[id][unit]) {
+			data[id][unit] = []
+		}
+		data[id][unit].push(value)
+	}
+
+	const readableStream = new Blob([JSON.stringify(data)]).stream()
+	const compressedStream = readableStream.pipeThrough(new CompressionStream('gzip'))
+	const arrayBuffer = await new Response(compressedStream).arrayBuffer()
+	fs.writeFile(path.join(basePath, 'metabase.json.gz'), Buffer.from(arrayBuffer))
+}
+
+await getMetabase()
+await getCatalogue()

js/data/meta/metabase.json.gz

@@ -133,6 +133,13 @@ export default class IOSelector {
 		}
 	}
 
+	clear() {
+		this._r.replaceChildren()
+		this._columns = []
+		this._object = []
+		this._target = -1
+	}
+
 	terminate() {
 		this._r.remove()
 	}

js/data/meta/package-lock.json

@@ -6,28 +6,30 @@ describe('classification', () => {
 	let page
 	beforeEach(async () => {
 		page = await getPage()
+		const dataSelectBox = page.locator('#ml_selector dl:first-child dd:nth-child(2) select')
+		await dataSelectBox.selectOption('eurostat')
 	})
 
 	afterEach(async () => {
 		await page?.close()
 	})
 
 	test('initialize', async () => {
-		const dataSelectBox = await page.waitForSelector('#ml_selector dl:first-child dd:nth-child(2) select')
-		await dataSelectBox.selectOption('eurostat')
+		const dataMenu = page.locator('#ml_selector #data_menu')
+		const themeList = dataMenu.locator('div:first-child')
 
-		const dataMenu = await page.waitForSelector('#ml_selector #data_menu')
-		const nameTextBox = await dataMenu.waitForSelector('select[name=name]')
-		const name = await (await nameTextBox.getProperty('value')).jsonValue()
-		expect(name).toBe('Population and employment')
-
-		const svg = await page.waitForSelector('#plot-area svg')
-		await svg.waitForSelector('.points .datas circle')
-		const size = (await svg.$$('.points .datas circle')).length
+		const svg = page.locator('#plot-area svg')
+		await svg.locator('.points .datas circle').first().waitFor()
+		const size = await svg.locator('.points .datas circle').count()
 		expect(size).toBeGreaterThan(0)
 
-		const aiManager = await getaimanager(page)
+		const aiManager = await getaimanager(page, {
+			ignoreProperties: ['_catalogue', '_metabase'],
+		})
 		expect(aiManager._datas).toBeDefined()
 		expect(aiManager._datas._x.length).toBe(size)
+
+		const nameTextBox = themeList.locator('select').last()
+		await expect(nameTextBox.inputValue()).resolves.toBe('nama_10_pe')
 	})
 })