fix: support device-pod filename rg
This commit is contained in:
@@ -142,7 +142,7 @@ test("device pod executor lists and isolates multiple logical device pods", asyn
|
||||
});
|
||||
|
||||
test("device-pod executor bounds gateway output text", async () => {
|
||||
const longText = "z".repeat(13000);
|
||||
const longText = "z".repeat(25000);
|
||||
const cloudApi = createServer(async (request, response) => {
|
||||
const body = await requestJson(request);
|
||||
response.writeHead(200, { "content-type": "application/json; charset=utf-8" });
|
||||
@@ -184,11 +184,12 @@ test("device-pod executor bounds gateway output text", async () => {
|
||||
});
|
||||
assert.equal(jobResponse.status, 202);
|
||||
const output = await waitForJobStatus(service.port, "device-pod-test", "job_bounded_output_test", "completed");
|
||||
assert.equal(output.output.text.length, 12000);
|
||||
assert.equal(output.text.length, 12000);
|
||||
assert.equal(output.bytes, 12000);
|
||||
assert.equal(output.output.text.length, 24000);
|
||||
assert.equal(output.text.length, 24000);
|
||||
assert.equal(output.bytes, 24000);
|
||||
assert.equal(output.truncation.truncated, true);
|
||||
assert.equal(output.truncation.originalBytes, 13000);
|
||||
assert.equal(output.truncation.originalBytes, 25000);
|
||||
assert.equal(output.output.omitted.reason, "device_job_output_truncated");
|
||||
assert.equal(JSON.stringify(output).includes(longText), false);
|
||||
} finally {
|
||||
await service.stop();
|
||||
|
||||
@@ -15,7 +15,7 @@ const environment = process.env.HWLAB_ENVIRONMENT || process.env.HWLAB_GITOPS_PR
|
||||
const cloudApiInternalUrl = normalizeBaseUrl(process.env.HWLAB_CLOUD_API_INTERNAL_URL || process.env.HWLAB_CLOUD_API_URL);
|
||||
const internalToken = textOr(process.env.HWLAB_DEVICE_POD_INTERNAL_TOKEN, "");
|
||||
const dispatchTimeoutMs = numberOr(process.env.HWLAB_DEVICE_POD_GATEWAY_DISPATCH_TIMEOUT_MS, 120000);
|
||||
const DEVICE_JOB_OUTPUT_MAX_BYTES = 12000;
|
||||
const DEVICE_JOB_OUTPUT_MAX_BYTES = 24000;
|
||||
const jobs = new Map();
|
||||
|
||||
listen(createServer(async (request, response) => {
|
||||
@@ -508,7 +508,7 @@ function deviceHostArgs(intent, args = {}) {
|
||||
}
|
||||
if (intent === "workspace.rg") {
|
||||
const pattern = textOr(args.pattern ?? args.query, "");
|
||||
return pattern ? ["workspace", "rg", pattern, textOr(args.path, "."), ...hostOptionArgs(args, ["glob", "g", "filesWithMatches", "l", "ignoreCase", "i", "maxCount"])] : null;
|
||||
return pattern ? ["workspace", "rg", pattern, textOr(args.path, "."), ...hostOptionArgs(args, ["glob", "g", "filesWithMatches", "nameOnly", "names", "l", "ignoreCase", "i", "maxCount", "limit"])] : null;
|
||||
}
|
||||
if (intent === "workspace.put") {
|
||||
const path = textOr(args.path, "");
|
||||
|
||||
@@ -2020,7 +2020,7 @@ test("cloud api dispatches authorized device jobs to the internal device-pod exe
|
||||
});
|
||||
|
||||
test("cloud api bounds device-pod job output payloads", async () => {
|
||||
const longText = "x".repeat(13000);
|
||||
const longText = "x".repeat(25000);
|
||||
const executor = createServer(async (request, response) => {
|
||||
const body = await requestJson(request);
|
||||
response.writeHead(200, { "content-type": "application/json; charset=utf-8" });
|
||||
@@ -2065,11 +2065,11 @@ test("cloud api bounds device-pod job output payloads", async () => {
|
||||
assert.equal(job.status, 200);
|
||||
const output = await getJson(port, `/v1/device-pods/device-pod-71-freq/jobs/${job.body.job.id}/output`, aliceLogin.cookie);
|
||||
assert.equal(output.status, 200);
|
||||
assert.equal(output.body.bytes, 12000);
|
||||
assert.equal(output.body.text.length, 12000);
|
||||
assert.equal(output.body.output.text.length, 12000);
|
||||
assert.equal(output.body.bytes, 24000);
|
||||
assert.equal(output.body.text.length, 24000);
|
||||
assert.equal(output.body.output.text.length, 24000);
|
||||
assert.equal(output.body.truncation.truncated, true);
|
||||
assert.equal(output.body.truncation.originalBytes, 13000);
|
||||
assert.equal(output.body.truncation.originalBytes, 25000);
|
||||
assert.equal(output.body.output.omitted.reason, "device_job_output_truncated");
|
||||
assert.equal(JSON.stringify(output.body).includes(longText), false);
|
||||
} finally {
|
||||
|
||||
@@ -202,7 +202,7 @@ const DEVICE_JOB_ACTIONABLE_INTENTS = new Set([
|
||||
"workspace.build",
|
||||
"debug.download"
|
||||
]);
|
||||
const DEVICE_JOB_OUTPUT_MAX_BYTES = 12000;
|
||||
const DEVICE_JOB_OUTPUT_MAX_BYTES = 24000;
|
||||
const DEVICE_POD_PROFILE_SECRET_KEY_PATTERN = /(?:token|secret|password|passphrase|credential|api[_-]?key|access[_-]?key|private[_-]?key|git[_-]?key|cloud[_-]?token|kubeconfig|database[_-]?url|db[_-]?url|connection[_-]?string)/iu;
|
||||
const DEVICE_POD_PROFILE_SECRET_VALUE_PATTERN = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|postgres(?:ql)?:\/\/|mysql:\/\/|mongodb(?:\+srv)?:\/\/|redis:\/\/|gh[pousr]_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,})/u;
|
||||
|
||||
|
||||
@@ -120,12 +120,18 @@ host `logTail` + `buildSummary` directly through the formal REST job path.
|
||||
### 3. `workspace rg --pattern "regex|pipes"` collides with local shell
|
||||
|
||||
When the regex contains `|` or other shell metacharacters, the local shell
|
||||
interprets them before `hwpod` sees the pattern. Two safe forms:
|
||||
interprets them before `hwpod` sees the pattern. `workspace rg` searches file
|
||||
contents by default; use `--name-only`, `--names`, or `--files-with-matches`
|
||||
when you need to match workspace-relative file names such as build artifacts.
|
||||
Safe forms:
|
||||
|
||||
```sh
|
||||
# shell-safe: pattern as one arg, but | still needs quoting
|
||||
hwpod <pod>:workspace:/ rg --pattern "Programming Done|Verify OK" --path <relpath>
|
||||
|
||||
# filename search for build artifacts
|
||||
hwpod <pod>:workspace:/ rg --pattern "FREQ_Controller_FW\\.(axf|bin|hex)$" --path projects/71-00075-11/FirmWare/MDK-ARM --name-only
|
||||
|
||||
# bullet-proof: pattern as base64
|
||||
P=$(printf "%s" "Error\(s\)|Warning\(s\)|Erase Done|Programming Done|Verify OK|Application running" | base64 -w0)
|
||||
hwpod <pod>:workspace:/ rg --pattern-b64 "$P" --path <relpath> --reason "log scan"
|
||||
@@ -171,6 +177,7 @@ Run read-only workspace, debug, and I/O jobs:
|
||||
hwpod device-pod-71-freq:workspace:/ ls
|
||||
hwpod device-pod-71-freq:workspace:/src rg main
|
||||
hwpod device-pod-71-freq:workspace:/ rg --pattern "Programming Done|Verify OK|Application running|Error" --path projects/01_baseline/captures
|
||||
hwpod device-pod-71-freq:workspace:/ rg --pattern "FREQ_Controller_FW\\.(axf|bin|hex)$" --path projects/71-00075-11/FirmWare/MDK-ARM --name-only
|
||||
hwpod device-pod-71-freq:debug-probe chip-id
|
||||
hwpod device-pod-71-freq:io-probe:/uart/1 read --duration-ms 5000
|
||||
```
|
||||
@@ -242,7 +249,7 @@ Do not treat local files such as `.device-pod/device-pod-71-freq.json` or `DEVIC
|
||||
- Use `hwpod`/`device-pod-cli` first for workspace, build, download, reset, UART, and debug-probe tasks.
|
||||
- After selecting a pod or resuming context, run `bootsharp --pod-id <devicePodId>` before edits, build, or download.
|
||||
- Prefer `workspace apply-patch` for source edits. Use `workspace put` only for intentional whole-file writes or new files, because it bypasses hunk-level conflict detection.
|
||||
- For regex searches with spaces, pipes, or multiple path tokens, prefer explicit `rg --pattern <regex> --path <workspaceRel>` or `--pattern-b64 <base64>` instead of positional shell parsing.
|
||||
- For regex searches with spaces, pipes, or multiple path tokens, prefer explicit `rg --pattern <regex> --path <workspaceRel>` or `--pattern-b64 <base64>` instead of positional shell parsing. `rg` searches file contents by default; add `--name-only` / `--names` / `--files-with-matches` to search file names, and `--limit <N>` to raise the file walk ceiling for large Keil workspaces.
|
||||
- If apply-patch fails, read the returned `patchHint`, re-read the current file with `workspace cat` or `workspace rg`, and retry a smaller exact-context hunk before using whole-file write.
|
||||
- Do not start by calling `/app/tools/hwlab-gateway-tran.mjs`, Windows-side Keil skills, serial-monitor skills, `keil-cli.py`, or generic shell when the Device Pod REST path is available.
|
||||
- If the CLI returns an auth, grant, lease, executor, or gateway blocker, fix that named blocker in the cloud-api/device-pod path; do not bypass the profile with generic gateway shell.
|
||||
|
||||
@@ -598,39 +598,48 @@ function workspaceRm(rel, options = {}) {
|
||||
ok('workspace.rm', { path: path.relative(profile.__workspaceRoot, file), removed: true, bytes: content.length, sha256: createHash('sha256').update(content).digest('hex') });
|
||||
}
|
||||
|
||||
function walkFiles(root, rel, results, limit = 5000) {
|
||||
if (results.length >= limit) return;
|
||||
function walkFiles(root, rel, results, limit = 20000) {
|
||||
if (results.length >= limit) return true;
|
||||
const dir = path.join(root, rel);
|
||||
let truncated = false;
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === '.git' || entry.name === 'node_modules' || entry.name === '.state') continue;
|
||||
const childRel = path.join(rel, entry.name);
|
||||
if (entry.isDirectory()) walkFiles(root, childRel, results, limit);
|
||||
if (entry.isDirectory()) truncated = walkFiles(root, childRel, results, limit) || truncated;
|
||||
else if (entry.isFile()) results.push(childRel);
|
||||
if (results.length >= limit) return;
|
||||
if (results.length >= limit) return true;
|
||||
}
|
||||
return truncated;
|
||||
}
|
||||
|
||||
function rgWorkspace(pattern, rel, options = {}) {
|
||||
const profile = loadProfile();
|
||||
const root = resolveWorkspacePath(profile, rel || '.');
|
||||
const files = [];
|
||||
const scanLimit = parsePositiveInt(optionValue(options, 'limit'), 20000, 'rg limit', 1, 100000);
|
||||
let filesTruncated = false;
|
||||
const stat = fs.statSync(root);
|
||||
if (stat.isFile()) files.push(path.relative(profile.__workspaceRoot, root));
|
||||
else {
|
||||
const nested = [];
|
||||
const rootRel = path.relative(profile.__workspaceRoot, root) || '.';
|
||||
walkFiles(root, '.', nested);
|
||||
filesTruncated = walkFiles(root, '.', nested, scanLimit);
|
||||
files.push(...nested.map((fileRel) => rootRel === '.' ? fileRel : path.join(rootRel, fileRel)));
|
||||
}
|
||||
const re = new RegExp(pattern, 'i');
|
||||
const matches = [];
|
||||
const fileMatches = [];
|
||||
const filesWithMatches = Boolean(options.l || options.filesWithMatches || options['files-with-matches'] || options._?.includes('-l'));
|
||||
const nameOnly = Boolean(filesWithMatches || options.nameOnly || options['name-only'] || options.names || options._?.includes('--name-only'));
|
||||
const globs = normalizeList(options.g).concat(normalizeList(options.glob), normalizeList(options.iglob));
|
||||
const types = normalizeList(options.t).concat(normalizeList(options.type));
|
||||
for (const fileRel of files) {
|
||||
if (!rgFileAllowed(fileRel, { globs, types })) continue;
|
||||
if (!filesWithMatches && matches.length >= 100) break;
|
||||
if (nameOnly) {
|
||||
if (re.test(fileRel.replace(/\\/gu, '/'))) fileMatches.push(fileRel);
|
||||
continue;
|
||||
}
|
||||
if (matches.length >= 100) break;
|
||||
const full = resolveWorkspacePath(profile, fileRel);
|
||||
const buffer = fs.readFileSync(full);
|
||||
if (buffer.includes(0)) continue;
|
||||
@@ -645,7 +654,7 @@ function rgWorkspace(pattern, rel, options = {}) {
|
||||
if (matches.length < 100) matches.push({ path: fileRel, line: index + 1, text: line.slice(0, 500) });
|
||||
}
|
||||
}
|
||||
ok('workspace.rg', { pattern, root: path.relative(profile.__workspaceRoot, root) || '.', count: filesWithMatches ? fileMatches.length : matches.length, files: filesWithMatches ? fileMatches : undefined, matches: filesWithMatches ? undefined : matches, options: { globs, types, filesWithMatches } });
|
||||
ok('workspace.rg', { pattern, root: path.relative(profile.__workspaceRoot, root) || '.', count: nameOnly ? fileMatches.length : matches.length, files: nameOnly ? fileMatches : undefined, matches: nameOnly ? undefined : matches, options: { globs, types, filesWithMatches, nameOnly, scanLimit, filesScanned: files.length, filesTruncated } });
|
||||
}
|
||||
|
||||
function resolveRgWorkspaceRequest(rest) {
|
||||
@@ -654,7 +663,14 @@ function resolveRgWorkspaceRequest(rest) {
|
||||
if (encodedPattern !== undefined) {
|
||||
return {
|
||||
pattern: decodeBase64Utf8(encodedPattern, '--pattern-b64'),
|
||||
rel: parsed._[0] || '.',
|
||||
rel: optionString(parsed, 'path') || parsed._[0] || '.',
|
||||
options: parsed,
|
||||
};
|
||||
}
|
||||
if (parsed.pattern !== undefined) {
|
||||
return {
|
||||
pattern: String(Array.isArray(parsed.pattern) ? parsed.pattern[0] : parsed.pattern),
|
||||
rel: optionString(parsed, 'path') || parsed._[0] || '.',
|
||||
options: parsed,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -597,6 +597,66 @@ test("device-pod-cli accepts explicit workspace rg pattern and path options", as
|
||||
});
|
||||
});
|
||||
|
||||
test("device-pod-cli forwards workspace rg filename search and scan limit options", async () => {
|
||||
const seen: any[] = [];
|
||||
const result = await runAssembledDevicePodCli([
|
||||
"D601-71-FREQ:workspace:/",
|
||||
"rg",
|
||||
"--pattern",
|
||||
"FREQ_Controller_FW\\\\.(axf|bin|hex)$",
|
||||
"--path",
|
||||
"projects/71-00075-11/FirmWare/MDK-ARM",
|
||||
"--name-only",
|
||||
"--files-with-matches",
|
||||
"--limit",
|
||||
"25000"
|
||||
], {
|
||||
fetchImpl: async (url, init) => {
|
||||
seen.push({ url: String(url), body: JSON.parse(String(init?.body ?? "{}")) });
|
||||
return jsonResponse(202, { ok: true, status: "running", job: { id: "job_rg_names", status: "running" } });
|
||||
},
|
||||
now: () => "2026-06-04T00:00:00.000Z"
|
||||
});
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.deepEqual(seen[0].body, {
|
||||
intent: "workspace.rg",
|
||||
args: {
|
||||
pattern: "FREQ_Controller_FW\\\\.(axf|bin|hex)$",
|
||||
path: "projects/71-00075-11/FirmWare/MDK-ARM",
|
||||
nameOnly: true,
|
||||
filesWithMatches: true,
|
||||
limit: 25000
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("device-pod-cli forwards workspace rg base64 patterns with explicit path", async () => {
|
||||
const seen: any[] = [];
|
||||
const pattern = Buffer.from("Error\\(s\\)|Warning\\(s\\)", "utf8").toString("base64");
|
||||
const result = await runAssembledDevicePodCli([
|
||||
"D601-71-FREQ:workspace:/",
|
||||
"rg",
|
||||
"--pattern-b64",
|
||||
pattern,
|
||||
"--path",
|
||||
"projects/71-00075-11/FirmWare/MDK-ARM"
|
||||
], {
|
||||
fetchImpl: async (url, init) => {
|
||||
seen.push({ url: String(url), body: JSON.parse(String(init?.body ?? "{}")) });
|
||||
return jsonResponse(202, { ok: true, status: "running", job: { id: "job_rg_b64", status: "running" } });
|
||||
},
|
||||
now: () => "2026-06-04T00:00:00.000Z"
|
||||
});
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.deepEqual(seen[0].body, {
|
||||
intent: "workspace.rg",
|
||||
args: {
|
||||
pattern: "Error\\(s\\)|Warning\\(s\\)",
|
||||
path: "projects/71-00075-11/FirmWare/MDK-ARM"
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function jsonResponse(status: number, body: unknown) {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { resolveRuntimeEndpoint, runtimeEndpointVisibility, sameRuntimeEndpointS
|
||||
const VERSION = "0.2.0-rest";
|
||||
const CLI_NAME = "device-pod-cli";
|
||||
const DEFAULT_TIMEOUT_MS = 30000;
|
||||
const BOOLEAN_OPTIONS = new Set(["dryRun", "full", "help", "h"]);
|
||||
const BOOLEAN_OPTIONS = new Set(["dryRun", "filesWithMatches", "full", "help", "h", "ignoreCase", "nameOnly", "names"]);
|
||||
|
||||
type FetchLike = typeof fetch;
|
||||
type EnvLike = Record<string, string | undefined>;
|
||||
@@ -250,10 +250,13 @@ async function buildJob(selector: any, operation: string, rest: string[], parsed
|
||||
"glob",
|
||||
"g",
|
||||
"filesWithMatches",
|
||||
"nameOnly",
|
||||
"names",
|
||||
"l",
|
||||
"ignoreCase",
|
||||
"i",
|
||||
"maxCount"
|
||||
"maxCount",
|
||||
"limit"
|
||||
]));
|
||||
}
|
||||
else if (operation === "put") {
|
||||
|
||||
Reference in New Issue
Block a user