Skip to content

Commit a1a2f14

Browse files
mstangemstange
committed
Add merge-android-profiles script
1 parent 687156c commit a1a2f14

File tree

3 files changed

+321
-0
lines changed

3 files changed

+321
-0
lines changed

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
"build-l10n-prod:quiet": "yarn build:clean && yarn build-photon && cross-env NODE_ENV=production L10N=1 webpack",
2020
"build-l10n-prod": "yarn build-l10n-prod:quiet --progress",
2121
"build-photon": "webpack --config res/photon/webpack.config.js",
22+
"build-merge-android-profiles": "yarn build-merge-android-profiles:quiet --progress",
23+
"build-merge-android-profiles:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/merge-android-profiles/webpack.config.js",
2224
"build-symbolicator-cli": "yarn build-symbolicator-cli:quiet --progress",
2325
"build-symbolicator-cli:quiet": "yarn build:clean && cross-env NODE_ENV=production webpack --config src/symbolicator-cli/webpack.config.js",
2426
"lint": "node bin/output-fixing-commands.js run-p lint-js lint-css prettier-run",

‎src/merge-android-profiles/index.ts‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
/**
2+
* Merge two existing profiles, taking the samples from the first profile and
3+
* the markers from the second profile.
4+
*
5+
* This was useful during early 2025 when the Mozilla Performance team was
6+
* doing a lot of Android startup profiling:
7+
*
8+
* - The "samples" profile would be collected using simpleperf and converted
9+
* with samply import.
10+
* - The "markers" profile would be collected using the Gecko profiler.
11+
*
12+
* To use this script, it first needs to be built:
13+
* yarn build-merge-android-profiles
14+
*
15+
* Then it can be run from the `dist` directory:
16+
* node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
17+
*
18+
* For example:
19+
* yarn build-merge-android-profiles && node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
20+
*
21+
*/
22+
23+
import fs from 'fs';
24+
import minimist from 'minimist';
25+
26+
import {
27+
unserializeProfileOfArbitraryFormat,
28+
adjustMarkerTimestamps,
29+
} from '../profile-logic/process-profile';
30+
import { getProfileUrlForHash } from '../actions/receive-profile';
31+
import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema';
32+
import { ensureExists } from '../utils/types';
33+
import { StringTable } from '../utils/string-table';
34+
35+
import type { Profile, RawThread, Tid } from '../types/profile';
36+
import { compress } from 'firefox-profiler/utils/gz';
37+
38+
type ProfileSource =
39+
| {
40+
type: 'HASH';
41+
hash: string;
42+
}
43+
| {
44+
type: 'FILE';
45+
file: string;
46+
};
47+
48+
interface CliOptions {
49+
samplesProf: ProfileSource;
50+
markersProf: ProfileSource;
51+
filterByProcessPrefix: string | undefined;
52+
assumeSamplesProfileHasStartTimeZero: boolean;
53+
outputFile: string;
54+
}
55+
56+
async function fetchProfileWithHash(hash: string): Promise<Profile> {
57+
const response = await fetch(getProfileUrlForHash(hash));
58+
const serializedProfile = await response.json();
59+
return unserializeProfileOfArbitraryFormat(serializedProfile);
60+
}
61+
62+
async function loadProfileFromFile(path: string): Promise<Profile> {
63+
const uint8Array = fs.readFileSync(path, null);
64+
return unserializeProfileOfArbitraryFormat(uint8Array.buffer);
65+
}
66+
67+
async function loadProfile(source: ProfileSource): Promise<Profile> {
68+
switch (source.type) {
69+
case 'HASH':
70+
return fetchProfileWithHash(source.hash);
71+
case 'FILE':
72+
return loadProfileFromFile(source.file);
73+
default:
74+
return source;
75+
}
76+
}
77+
78+
export async function run(options: CliOptions) {
79+
const profileWithSamples: Profile = await loadProfile(options.samplesProf);
80+
const profileWithMarkers: Profile = await loadProfile(options.markersProf);
81+
82+
// const referenceSampleTime = 169912951.547432; // filteredThread.samples.time[0] after zooming in on samples in mozilla::dom::indexedDB::BackgroundTransactionChild::RecvComplete
83+
// const referenceMarkerTime = 664.370158 ; // selectedMarker.start after selecting the marker for the "complete" DOMEvent
84+
85+
// console.log(profileWithSamples.meta);
86+
// console.log(profileWithMarkers.meta);
87+
88+
if (
89+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot ===
90+
undefined &&
91+
options.assumeSamplesProfileHasStartTimeZero
92+
) {
93+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 0;
94+
}
95+
96+
let timeDelta =
97+
profileWithMarkers.meta.startTime - profileWithSamples.meta.startTime;
98+
if (
99+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
100+
undefined &&
101+
profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
102+
undefined
103+
) {
104+
timeDelta =
105+
(profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot -
106+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot) /
107+
1000000;
108+
}
109+
110+
// console.log({ timeDelta });
111+
112+
const profile = profileWithSamples;
113+
profile.meta.markerSchema = profileWithMarkers.meta.markerSchema;
114+
profile.pages = profileWithMarkers.pages;
115+
116+
const markerProfileCategoryToCategory = new Map();
117+
const markerProfileCategories = ensureExists(
118+
profileWithMarkers.meta.categories
119+
);
120+
const profileCategories = ensureExists(profile.meta.categories);
121+
for (
122+
let markerCategoryIndex = 0;
123+
markerCategoryIndex < markerProfileCategories.length;
124+
markerCategoryIndex++
125+
) {
126+
const category = markerProfileCategories[markerCategoryIndex];
127+
let categoryIndex = profileCategories.findIndex(
128+
(c) => c.name === category.name
129+
);
130+
if (categoryIndex === -1) {
131+
categoryIndex = profileCategories.length;
132+
profileCategories[categoryIndex] = {
133+
name: category.name,
134+
color: category.color,
135+
subcategories: ['Other'],
136+
};
137+
}
138+
markerProfileCategoryToCategory.set(markerCategoryIndex, categoryIndex);
139+
}
140+
141+
const markerThreadsByTid = new Map<Tid, RawThread>(
142+
profileWithMarkers.threads.map((thread) => ['' + thread.tid, thread])
143+
);
144+
// console.log([...markerThreadsByTid.keys()]);
145+
146+
// console.log(profile.threads.map((thread) => thread.tid));
147+
148+
const stringIndexMarkerFieldsByDataType =
149+
computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema);
150+
151+
const sampleThreadTidsWithoutCorrespondingMarkerThreads = new Set();
152+
153+
const stringTable = StringTable.withBackingArray(profile.shared.stringArray);
154+
const markerStringArray = profileWithMarkers.shared.stringArray;
155+
const keptThreads = [];
156+
for (const thread of profile.threads) {
157+
if (options.filterByProcessPrefix !== undefined) {
158+
if (!thread.processName!.startsWith(options.filterByProcessPrefix)) {
159+
continue;
160+
}
161+
}
162+
keptThreads.push(thread);
163+
const tid = thread.tid;
164+
const markerThread = markerThreadsByTid.get(tid);
165+
if (markerThread === undefined) {
166+
sampleThreadTidsWithoutCorrespondingMarkerThreads.add(tid);
167+
continue;
168+
}
169+
markerThreadsByTid.delete(tid);
170+
171+
thread.markers = adjustMarkerTimestamps(markerThread.markers, timeDelta);
172+
for (let i = 0; i < thread.markers.length; i++) {
173+
thread.markers.category[i] = ensureExists(
174+
markerProfileCategoryToCategory.get(thread.markers.category[i])
175+
);
176+
thread.markers.name[i] = stringTable.indexForString(
177+
markerStringArray[thread.markers.name[i]]
178+
);
179+
const data = thread.markers.data[i];
180+
if (data !== null && data.type) {
181+
const markerType = data.type;
182+
const stringIndexMarkerFields =
183+
stringIndexMarkerFieldsByDataType.get(markerType);
184+
if (stringIndexMarkerFields !== undefined) {
185+
for (const fieldKey of stringIndexMarkerFields) {
186+
const stringIndex = (data as any)[fieldKey];
187+
if (typeof stringIndex === 'number') {
188+
const newStringIndex = stringTable.indexForString(
189+
markerStringArray[stringIndex]
190+
);
191+
(data as any)[fieldKey] = newStringIndex;
192+
}
193+
}
194+
}
195+
}
196+
}
197+
}
198+
199+
profile.threads = keptThreads;
200+
201+
// console.log(
202+
// `Have ${markerThreadsByTid.size} marker threads left over which weren't slurped up by sample threads:`,
203+
// [...markerThreadsByTid.keys()]
204+
// );
205+
// if (markerThreadsByTid.size !== 0) {
206+
// console.log(
207+
// `Have ${sampleThreadTidsWithoutCorrespondingMarkerThreads.size} sample threads which didn't find corresponding marker threads:`,
208+
// [...sampleThreadTidsWithoutCorrespondingMarkerThreads]
209+
// );
210+
// }
211+
212+
if (options.outputFile.endsWith('.gz')) {
213+
fs.writeFileSync(
214+
options.outputFile,
215+
await compress(JSON.stringify(profile))
216+
);
217+
} else {
218+
fs.writeFileSync(options.outputFile, JSON.stringify(profile));
219+
}
220+
}
221+
222+
export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
223+
const argv = minimist(processArgv.slice(2));
224+
225+
const hasSamplesHash =
226+
'samples-hash' in argv && typeof argv['samples-hash'] === 'string';
227+
const hasSamplesFile =
228+
'samples-file' in argv && typeof argv['samples-file'] === 'string';
229+
const hasMarkersHash =
230+
'markers-hash' in argv && typeof argv['markers-hash'] === 'string';
231+
const hasMarkersFile =
232+
'markers-file' in argv && typeof argv['markers-file'] === 'string';
233+
234+
if (!hasSamplesHash && !hasSamplesFile) {
235+
throw new Error('Either --samples-file or --samples-hash must be supplied');
236+
}
237+
if (hasSamplesHash && hasSamplesFile) {
238+
throw new Error(
239+
'Only one of --samples-file or --samples-hash can be supplied'
240+
);
241+
}
242+
if (!hasMarkersHash && !hasMarkersFile) {
243+
throw new Error('Either --markers-file or --markers-hash must be supplied');
244+
}
245+
if (hasMarkersHash && hasMarkersFile) {
246+
throw new Error(
247+
'Only one of --markers-file or --markers-hash can be supplied'
248+
);
249+
}
250+
251+
const samplesProf: ProfileSource = hasSamplesHash
252+
? { type: 'HASH', hash: argv['samples-hash'] }
253+
: { type: 'FILE', file: argv['samples-file'] };
254+
const markersProf: ProfileSource = hasMarkersHash
255+
? { type: 'HASH', hash: argv['markers-hash'] }
256+
: { type: 'FILE', file: argv['markers-file'] };
257+
258+
return {
259+
samplesProf,
260+
markersProf,
261+
filterByProcessPrefix: argv['filter-by-process-prefix'],
262+
assumeSamplesProfileHasStartTimeZero: 'assume-samply' in argv,
263+
outputFile: argv['output-file'],
264+
};
265+
}
266+
267+
if (!module.parent) {
268+
try {
269+
const options = makeOptionsFromArgv(process.argv);
270+
run(options).catch((err) => {
271+
throw err;
272+
});
273+
} catch (e) {
274+
console.error(e);
275+
process.exit(1);
276+
}
277+
}

‎src/merge-android-profiles/webpack.config.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
const path = require('path');
2+
const projectRoot = path.join(__dirname, '../..');
3+
const includes = [path.join(projectRoot, 'src')];
4+
5+
module.exports = {
6+
name: 'merge-android-profiles',
7+
target: 'node',
8+
mode: process.env.NODE_ENV,
9+
resolve: {
10+
extensions: ['.js', '.jsx', '.ts', '.tsx'],
11+
alias: {
12+
'firefox-profiler-res': path.resolve(projectRoot, 'res'),
13+
},
14+
},
15+
output: {
16+
path: path.resolve(projectRoot, 'dist'),
17+
filename: 'merge-android-profiles.js',
18+
},
19+
entry: './src/merge-android-profiles/index.ts',
20+
module: {
21+
rules: [
22+
{
23+
test: /\.(ts|tsx)$/,
24+
use: ['babel-loader'],
25+
include: includes,
26+
},
27+
{
28+
test: /\.js$/,
29+
include: path.resolve(projectRoot, 'res'),
30+
type: 'asset/resource',
31+
},
32+
{
33+
test: /\.svg$/,
34+
type: 'asset/resource',
35+
},
36+
],
37+
},
38+
experiments: {
39+
// Make WebAssembly work just like in webpack v4
40+
syncWebAssembly: true,
41+
},
42+
};

0 commit comments

Comments
 (0)