From a0ca05278dfaf8442f54419f18908d487784b0a2 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Tue, 6 May 2025 15:04:46 -0400 Subject: [PATCH] Add merge-android-profiles script --- package.json | 2 + src/merge-android-profiles/index.ts | 277 +++++++++++++++++++ src/merge-android-profiles/webpack.config.js | 42 +++ 3 files changed, 321 insertions(+) create mode 100644 src/merge-android-profiles/index.ts create mode 100644 src/merge-android-profiles/webpack.config.js diff --git a/package.json b/package.json index d72ca7c14a..6fe0529df6 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "build-l10n-prod:quiet": "yarn build:clean && yarn build-photon && cross-env NODE_ENV=production L10N=1 webpack", "build-l10n-prod": "yarn build-l10n-prod:quiet --progress", "build-photon": "webpack --config res/photon/webpack.config.js", + "build-merge-android-profiles": "yarn build-merge-android-profiles:quiet --progress", + "build-merge-android-profiles:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/merge-android-profiles/webpack.config.js", "build-symbolicator-cli": "yarn build-symbolicator-cli:quiet --progress", "build-symbolicator-cli:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/symbolicator-cli/webpack.config.js", "lint": "node bin/output-fixing-commands.js run-p lint-js lint-css prettier-run", diff --git a/src/merge-android-profiles/index.ts b/src/merge-android-profiles/index.ts new file mode 100644 index 0000000000..d986dc5065 --- /dev/null +++ b/src/merge-android-profiles/index.ts @@ -0,0 +1,277 @@ +/** + * Merge two existing profiles, taking the samples from the first profile and + * the markers from the second profile. + * + * This was useful during early 2025 when the Mozilla Performance team was + * doing a lot of Android startup profiling: + * + * - The "samples" profile would be collected using simpleperf and converted + * with samply import. + * - The "markers" profile would be collected using the Gecko profiler. + * + * To use this script, it first needs to be built: + * yarn build-merge-android-profiles + * + * Then it can be run from the `dist` directory: + * node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json + * + * For example: + * yarn build-merge-android-profiles && node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json + * + */ + +import fs from 'fs'; +import minimist from 'minimist'; + +import { + unserializeProfileOfArbitraryFormat, + adjustMarkerTimestamps, +} from '../profile-logic/process-profile'; +import { getProfileUrlForHash } from '../actions/receive-profile'; +import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema'; +import { ensureExists } from '../utils/types'; +import { StringTable } from '../utils/string-table'; + +import type { Profile, RawThread, Tid } from '../types/profile'; +import { compress } from 'firefox-profiler/utils/gz'; + +type ProfileSource = + | { + type: 'HASH'; + hash: string; + } + | { + type: 'FILE'; + file: string; + }; + +interface CliOptions { + samplesProf: ProfileSource; + markersProf: ProfileSource; + filterByProcessPrefix: string | undefined; + assumeSamplesProfileHasStartTimeZero: boolean; + outputFile: string; +} + +async function fetchProfileWithHash(hash: string): Promise { + const response = await fetch(getProfileUrlForHash(hash)); + const serializedProfile = await response.json(); + return unserializeProfileOfArbitraryFormat(serializedProfile); +} + +async function loadProfileFromFile(path: string): Promise { + const uint8Array = fs.readFileSync(path, null); + return unserializeProfileOfArbitraryFormat(uint8Array.buffer); +} + +async function loadProfile(source: ProfileSource): Promise { + switch (source.type) { + case 'HASH': + return fetchProfileWithHash(source.hash); + case 'FILE': + return loadProfileFromFile(source.file); + default: + return source; + } +} + +export async function run(options: CliOptions) { + const profileWithSamples: Profile = await loadProfile(options.samplesProf); + const profileWithMarkers: Profile = await loadProfile(options.markersProf); + + // const referenceSampleTime = 169912951.547432; // filteredThread.samples.time[0] after zooming in on samples in mozilla::dom::indexedDB::BackgroundTransactionChild::RecvComplete + // const referenceMarkerTime = 664.370158 ; // selectedMarker.start after selecting the marker for the "complete" DOMEvent + + // console.log(profileWithSamples.meta); + // console.log(profileWithMarkers.meta); + + if ( + profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot === + undefined && + options.assumeSamplesProfileHasStartTimeZero + ) { + profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 0; + } + + let timeDelta = + profileWithMarkers.meta.startTime - profileWithSamples.meta.startTime; + if ( + profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !== + undefined && + profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !== + undefined + ) { + timeDelta = + (profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot - + profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot) / + 1000000; + } + + // console.log({ timeDelta }); + + const profile = profileWithSamples; + profile.meta.markerSchema = profileWithMarkers.meta.markerSchema; + profile.pages = profileWithMarkers.pages; + + const markerProfileCategoryToCategory = new Map(); + const markerProfileCategories = ensureExists( + profileWithMarkers.meta.categories + ); + const profileCategories = ensureExists(profile.meta.categories); + for ( + let markerCategoryIndex = 0; + markerCategoryIndex < markerProfileCategories.length; + markerCategoryIndex++ + ) { + const category = markerProfileCategories[markerCategoryIndex]; + let categoryIndex = profileCategories.findIndex( + (c) => c.name === category.name + ); + if (categoryIndex === -1) { + categoryIndex = profileCategories.length; + profileCategories[categoryIndex] = { + name: category.name, + color: category.color, + subcategories: ['Other'], + }; + } + markerProfileCategoryToCategory.set(markerCategoryIndex, categoryIndex); + } + + const markerThreadsByTid = new Map( + profileWithMarkers.threads.map((thread) => ['' + thread.tid, thread]) + ); + // console.log([...markerThreadsByTid.keys()]); + + // console.log(profile.threads.map((thread) => thread.tid)); + + const stringIndexMarkerFieldsByDataType = + computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema); + + const sampleThreadTidsWithoutCorrespondingMarkerThreads = new Set(); + + const stringTable = StringTable.withBackingArray(profile.shared.stringArray); + const markerStringArray = profileWithMarkers.shared.stringArray; + const keptThreads = []; + for (const thread of profile.threads) { + if (options.filterByProcessPrefix !== undefined) { + if (!thread.processName!.startsWith(options.filterByProcessPrefix)) { + continue; + } + } + keptThreads.push(thread); + const tid = thread.tid; + const markerThread = markerThreadsByTid.get(tid); + if (markerThread === undefined) { + sampleThreadTidsWithoutCorrespondingMarkerThreads.add(tid); + continue; + } + markerThreadsByTid.delete(tid); + + thread.markers = adjustMarkerTimestamps(markerThread.markers, timeDelta); + for (let i = 0; i < thread.markers.length; i++) { + thread.markers.category[i] = ensureExists( + markerProfileCategoryToCategory.get(thread.markers.category[i]) + ); + thread.markers.name[i] = stringTable.indexForString( + markerStringArray[thread.markers.name[i]] + ); + const data = thread.markers.data[i]; + if (data !== null && data.type) { + const markerType = data.type; + const stringIndexMarkerFields = + stringIndexMarkerFieldsByDataType.get(markerType); + if (stringIndexMarkerFields !== undefined) { + for (const fieldKey of stringIndexMarkerFields) { + const stringIndex = (data as any)[fieldKey]; + if (typeof stringIndex === 'number') { + const newStringIndex = stringTable.indexForString( + markerStringArray[stringIndex] + ); + (data as any)[fieldKey] = newStringIndex; + } + } + } + } + } + } + + profile.threads = keptThreads; + + // console.log( + // `Have ${markerThreadsByTid.size} marker threads left over which weren't slurped up by sample threads:`, + // [...markerThreadsByTid.keys()] + // ); + // if (markerThreadsByTid.size !== 0) { + // console.log( + // `Have ${sampleThreadTidsWithoutCorrespondingMarkerThreads.size} sample threads which didn't find corresponding marker threads:`, + // [...sampleThreadTidsWithoutCorrespondingMarkerThreads] + // ); + // } + + if (options.outputFile.endsWith('.gz')) { + fs.writeFileSync( + options.outputFile, + await compress(JSON.stringify(profile)) + ); + } else { + fs.writeFileSync(options.outputFile, JSON.stringify(profile)); + } +} + +export function makeOptionsFromArgv(processArgv: string[]): CliOptions { + const argv = minimist(processArgv.slice(2)); + + const hasSamplesHash = + 'samples-hash' in argv && typeof argv['samples-hash'] === 'string'; + const hasSamplesFile = + 'samples-file' in argv && typeof argv['samples-file'] === 'string'; + const hasMarkersHash = + 'markers-hash' in argv && typeof argv['markers-hash'] === 'string'; + const hasMarkersFile = + 'markers-file' in argv && typeof argv['markers-file'] === 'string'; + + if (!hasSamplesHash && !hasSamplesFile) { + throw new Error('Either --samples-file or --samples-hash must be supplied'); + } + if (hasSamplesHash && hasSamplesFile) { + throw new Error( + 'Only one of --samples-file or --samples-hash can be supplied' + ); + } + if (!hasMarkersHash && !hasMarkersFile) { + throw new Error('Either --markers-file or --markers-hash must be supplied'); + } + if (hasMarkersHash && hasMarkersFile) { + throw new Error( + 'Only one of --markers-file or --markers-hash can be supplied' + ); + } + + const samplesProf: ProfileSource = hasSamplesHash + ? { type: 'HASH', hash: argv['samples-hash'] } + : { type: 'FILE', file: argv['samples-file'] }; + const markersProf: ProfileSource = hasMarkersHash + ? { type: 'HASH', hash: argv['markers-hash'] } + : { type: 'FILE', file: argv['markers-file'] }; + + return { + samplesProf, + markersProf, + filterByProcessPrefix: argv['filter-by-process-prefix'], + assumeSamplesProfileHasStartTimeZero: 'assume-samply' in argv, + outputFile: argv['output-file'], + }; +} + +if (!module.parent) { + try { + const options = makeOptionsFromArgv(process.argv); + run(options).catch((err) => { + throw err; + }); + } catch (e) { + console.error(e); + process.exit(1); + } +} diff --git a/src/merge-android-profiles/webpack.config.js b/src/merge-android-profiles/webpack.config.js new file mode 100644 index 0000000000..669dcf4c01 --- /dev/null +++ b/src/merge-android-profiles/webpack.config.js @@ -0,0 +1,42 @@ +const path = require('path'); +const projectRoot = path.join(__dirname, '../..'); +const includes = [path.join(projectRoot, 'src')]; + +module.exports = { + name: 'merge-android-profiles', + target: 'node', + mode: process.env.NODE_ENV, + resolve: { + extensions: ['.js', '.jsx', '.ts', '.tsx'], + alias: { + 'firefox-profiler-res': path.resolve(projectRoot, 'res'), + }, + }, + output: { + path: path.resolve(projectRoot, 'dist'), + filename: 'merge-android-profiles.js', + }, + entry: './src/merge-android-profiles/index.ts', + module: { + rules: [ + { + test: /\.(ts|tsx)$/, + use: ['babel-loader'], + include: includes, + }, + { + test: /\.js$/, + include: path.resolve(projectRoot, 'res'), + type: 'asset/resource', + }, + { + test: /\.svg$/, + type: 'asset/resource', + }, + ], + }, + experiments: { + // Make WebAssembly work just like in webpack v4 + syncWebAssembly: true, + }, +};