diff --git a/.changeset/curvy-clouds-drive.md b/.changeset/curvy-clouds-drive.md new file mode 100644 index 00000000..6c09cc8d --- /dev/null +++ b/.changeset/curvy-clouds-drive.md @@ -0,0 +1,6 @@ +--- +"@nodesecure/contact": minor +"@nodesecure/scanner": minor +--- + +feat(scanner): add highlighted packages and contacts extractors diff --git a/workspaces/contact/src/ContactExtractor.class.ts b/workspaces/contact/src/ContactExtractor.class.ts index c71a97ee..e99f423c 100644 --- a/workspaces/contact/src/ContactExtractor.class.ts +++ b/workspaces/contact/src/ContactExtractor.class.ts @@ -15,7 +15,7 @@ export type { }; export interface ContactExtractorPackageMetadata { - author?: Contact; + author?: Contact | null; maintainers: Contact[]; } @@ -125,7 +125,7 @@ export class ContactExtractor { } } -function extractMetadataContacts( +export function extractMetadataContacts( metadata: ContactPackageMetaData ): Contact[] { return [ diff --git a/workspaces/contact/src/index.ts b/workspaces/contact/src/index.ts index d1ba24ab..f1e384e9 100644 --- a/workspaces/contact/src/index.ts +++ b/workspaces/contact/src/index.ts @@ -3,3 +3,4 @@ export { compareContact } from "./utils/index.ts"; export { NsResolver } from "./NsResolver.class.ts"; +export { UnlitContact } from "./UnlitContact.class.ts"; diff --git a/workspaces/scanner/docs/extractors.md b/workspaces/scanner/docs/extractors.md index 5a0e12d5..6856108a 100644 --- a/workspaces/scanner/docs/extractors.md +++ b/workspaces/scanner/docs/extractors.md @@ -39,6 +39,10 @@ Available probes include: | Warnings | manifest | | Extentions | manifest | | NodeDependencies | manifest | +| HighlightedPackages | manifest | +| HighlightedContacts | packument | + +## ProbeExtractor All probes follow the same `ProbeExtractor` interface, which acts as an iterator-like contract: diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index ad4af65c..ed46911d 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -17,12 +17,11 @@ import { } from "@nodesecure/mama"; import { getNpmRegistryURL } from "@nodesecure/npm-registry-sdk"; import type Config from "@npmcli/config"; -import semver from "semver"; // Import Internal Dependencies import { - getDependenciesWarnings, addMissingVersionFlags, + getDependenciesWarnings, getUsedDeps, getManifestLinks, NPM_TOKEN @@ -48,7 +47,7 @@ import type { Options, Payload } from "./types.ts"; -import { parseSemverRange } from "./utils/parseSemverRange.ts"; +import { HighlightedPackages } from "./extractors/probes/HighlightedPackagesExtractor.class.ts"; // CONSTANTS const kDefaultDependencyVersionFields = { @@ -185,7 +184,6 @@ export async function depWalker( }; const dependencies: Map = new Map(); - const highlightedPackages: Set = new Set(); const identifiersToHighlight = new Set(options.highlight?.identifiers ?? []); const npmTreeWalker = new npm.TreeWalker({ registry, @@ -363,6 +361,7 @@ export async function depWalker( // We do this because it "seem" impossible to link all dependencies in the first walk. // Because we are dealing with package only one time it may happen sometimes. const globalWarnings: GlobalWarning[] = []; + const highlightedPackagesExtractor = new HighlightedPackages(options.highlight?.packages ?? {}); for (const [packageName, dependency] of dependencies) { const metadataIntegrities = dependency.metadata?.integrity ?? {}; @@ -388,22 +387,12 @@ export async function depWalker( }); } } - const semverRanges = parseSemverRange(options.highlight?.packages ?? {}); for (const version of Object.entries(dependency.versions)) { const [verStr, verDescriptor] = version as [string, DependencyVersion]; - const packageRange = semverRanges?.[packageName]; - const org = parseNpmSpec(packageName)?.org; - const isScopeHighlighted = org !== null && `@${org}` in semverRanges; - - if ( - (packageRange && semver.satisfies(verStr, packageRange)) || - isScopeHighlighted - ) { - highlightedPackages.add(`${packageName}@${verStr}`); - } verDescriptor.flags.push( ...addMissingVersionFlags(new Set(verDescriptor.flags), dependency) ); + highlightedPackagesExtractor.next(verStr, verDescriptor, { name: packageName, dependency }); if (isLocalManifest(verDescriptor, mama, packageName)) { const author = mama.author; @@ -439,9 +428,10 @@ export async function depWalker( isRemoteScanning ); payload.warnings = globalWarnings.concat(dependencyConfusionWarnings as GlobalWarning[]).concat(warnings); + const { highlightedPackages } = highlightedPackagesExtractor.done(); payload.highlighted = { contacts: illuminated, - packages: [...highlightedPackages], + packages: highlightedPackages, identifiers: extractHighlightedIdentifiers(collectables, identifiersToHighlight) }; payload.dependencies = Object.fromEntries(dependencies); diff --git a/workspaces/scanner/src/extractors/probes/HighlightedContactsExtractor.class.ts b/workspaces/scanner/src/extractors/probes/HighlightedContactsExtractor.class.ts new file mode 100644 index 00000000..e50e3c36 --- /dev/null +++ b/workspaces/scanner/src/extractors/probes/HighlightedContactsExtractor.class.ts @@ -0,0 +1,50 @@ +// Import Third-party Dependencies +import { type EnforcedContact, type IlluminatedContact, UnlitContact, extractMetadataContacts } from "@nodesecure/contact"; +import type { Contact } from "@nodesecure/npm-types"; + +// Import Internal Dependencies +import type { + PackumentProbeExtractor +} from "../payload.ts"; +import type { Dependency } from "../../types.ts"; + +export type HighlightedContactsResult = { + illuminated: IlluminatedContact[]; +}; + +export class HighlightedContacts implements PackumentProbeExtractor { + level = "packument" as const; + + #unlitContacts: UnlitContact[]; + + constructor(contacts: EnforcedContact[]) { + this.#unlitContacts = contacts.map((contact) => new UnlitContact(contact)); + } + + next(packageName: string, dependency: Dependency) { + const extractedContacts = extractMetadataContacts(dependency.metadata); + this.addDependencyToUnlitContacts(extractedContacts, packageName); + } + + private addDependencyToUnlitContacts( + contacts: Contact[], + packageName: string + ) { + for (const unlit of this.#unlitContacts) { + const isMaintainer = contacts.some((contact) => unlit.compareTo(contact)); + if (isMaintainer) { + unlit.dependencies.add(packageName); + } + } + } + + done() { + const illuminated = this.#unlitContacts.flatMap( + (unlit) => (unlit.dependencies.size > 0 ? [unlit.illuminate()] : []) + ); + + return { + illuminated + }; + } +} diff --git a/workspaces/scanner/src/extractors/probes/HighlightedPackagesExtractor.class.ts b/workspaces/scanner/src/extractors/probes/HighlightedPackagesExtractor.class.ts new file mode 100644 index 00000000..fa3a5eb5 --- /dev/null +++ b/workspaces/scanner/src/extractors/probes/HighlightedPackagesExtractor.class.ts @@ -0,0 +1,89 @@ +// Import Third-party Dependencies +import { parseNpmSpec } from "@nodesecure/mama"; +import semver from "semver"; + +// Import Internal Dependencies +import type { + ManifestProbeExtractor, + ProbeExtractorManifestParent +} from "../payload.ts"; +import type { DependencyVersion, HighlightPackages } from "../../types.ts"; + +export type HighlightedPackagesResult = { + highlightedPackages: string[]; +}; + +export class HighlightedPackages implements ManifestProbeExtractor { + level = "manifest" as const; + #semverRanges: Record; + #highlightedPackages = new Set(); + + constructor(packages: HighlightPackages) { + this.#semverRanges = this.#parseSemverRange(packages); + } + + #parseSemverRange(packages: HighlightPackages) { + const pkgs = Array.isArray(packages) ? this.#parseSpecs(packages) : packages; + + return Object.entries(pkgs).reduce((acc, [name, semverRange]) => { + if (Array.isArray(semverRange)) { + acc[name] = semverRange.join(" || "); + } + else { + acc[name] = semverRange; + } + + return acc; + }, {}); + } + + #parseSpecs(specs: string[]) { + return specs.reduce((acc, spec) => { + // Handle scope-only entries like "@fastify", matching all packages under that scope + if (/^@[^/@]+$/.test(spec)) { + acc[spec] = ["*"]; + + return acc; + } + + const parsedSpec = parseNpmSpec(spec); + if (!parsedSpec) { + return acc; + } + const { name, semver } = parsedSpec; + const version = semver || "*"; + if (name in acc) { + acc[name].push(version); + } + else { + acc[name] = [version]; + } + + return acc; + }, {}); + } + + next( + version: string, + _: DependencyVersion, + parent: ProbeExtractorManifestParent + ) { + const packageRange = this.#semverRanges?.[parent.name]; + const org = parseNpmSpec(parent.name)?.org; + const isScopeHighlighted = org !== null && `@${org}` in this.#semverRanges; + + if ( + (packageRange && semver.satisfies(version, packageRange)) || + isScopeHighlighted + ) { + this.#highlightedPackages.add(`${parent.name}@${version}`); + } + } + + done() { + return { + highlightedPackages: [...this.#highlightedPackages] + }; + } +} + diff --git a/workspaces/scanner/src/extractors/probes/index.ts b/workspaces/scanner/src/extractors/probes/index.ts index 8bd4c9d6..700fd6fb 100644 --- a/workspaces/scanner/src/extractors/probes/index.ts +++ b/workspaces/scanner/src/extractors/probes/index.ts @@ -6,3 +6,5 @@ export * from "./VulnerabilitiesExtractor.class.ts"; export * from "./FlagsExtractor.class.ts"; export * from "./ExtensionsExtractor.class.ts"; export * from "./NodeDependenciesExtractor.class.ts"; +export * from "./HighlightedPackagesExtractor.class.ts"; +export * from "./HighlightedContactsExtractor.class.ts"; diff --git a/workspaces/scanner/src/utils/parseSemverRange.ts b/workspaces/scanner/src/utils/parseSemverRange.ts deleted file mode 100644 index e8f234f6..00000000 --- a/workspaces/scanner/src/utils/parseSemverRange.ts +++ /dev/null @@ -1,47 +0,0 @@ -// Import Third-party Dependencies -import { parseNpmSpec } from "@nodesecure/mama"; - -// Import Internal Dependencies -import { type HighlightPackages } from "../types.ts"; - -export function parseSemverRange(packages: HighlightPackages) { - const pkgs = Array.isArray(packages) ? parseSpecs(packages) : packages; - - return Object.entries(pkgs).reduce((acc, [name, semverRange]) => { - if (Array.isArray(semverRange)) { - acc[name] = semverRange.join(" || "); - } - else { - acc[name] = semverRange; - } - - return acc; - }, {}); -} - -function parseSpecs(specs: string[]) { - return specs.reduce((acc, spec) => { - // Handle scope-only entries like "@fastify", matching all packages under that scope - if (/^@[^/@]+$/.test(spec)) { - acc[spec] = ["*"]; - - return acc; - } - - const parsedSpec = parseNpmSpec(spec); - if (!parsedSpec) { - return acc; - } - const { name, semver } = parsedSpec; - const version = semver || "*"; - if (name in acc) { - acc[name].push(version); - } - else { - acc[name] = [version]; - } - - return acc; - }, {}); -} - diff --git a/workspaces/scanner/test/extractors/payload.spec.ts b/workspaces/scanner/test/extractors/payload.spec.ts index 97b97f4b..0a906174 100644 --- a/workspaces/scanner/test/extractors/payload.spec.ts +++ b/workspaces/scanner/test/extractors/payload.spec.ts @@ -1,9 +1,13 @@ +/* eslint-disable max-lines */ // Import Node.js Dependencies import assert from "node:assert"; import fs from "node:fs"; import path from "node:path"; import { describe, it } from "node:test"; +// Import Third-party Dependencies +import type { Contact } from "@nodesecure/npm-types"; + // Import Internal Dependencies import { Extractors, @@ -251,6 +255,368 @@ describe("Extractors.Probes", () => { }); }); +describe("HihglightedContacts", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + express: { + metadata: { + author: { name: "TJ Holowaychuk", email: "tj@vision-media.ca" }, + maintainers: [ + { name: "wesleytodd", email: "wes@wesleytodd.com" }, + { name: "jonchurch", email: "npm@jonchurch.com" }, + { name: "ctcpip", email: "c@labsector.com" }, + { name: "sheplu", email: "jean.burellier@gmail.com" } + ], + publishers: [] + }, + versions: { + "5.1.0": { + author: { name: "TJ Holowaychuk", email: "tj@vision-media.ca" } + } + }, + vulnerabilities: [] + } + } + }; + it("Given a contact with no name, it should not throw an Error", () => { + const highlighted = { + email: "foobar@gmail.com" + }; + + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedContacts( + [highlighted] + ) + ]); + + extractor.extractAndMerge(); + }); + + it(`Given dependencies where the Highlighted Contact doesn't appears, + it must return an empty Array`, () => { + const highlighted: Contact = { + name: "Ciaran Jessup" + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedContacts( + [highlighted] + ) + ]); + + const { illuminated } = extractor.extractAndMerge(); + assert.strictEqual(illuminated.length, 0); + }); + + it(`Given dependencies where the Highlighted Contact is the author of the package, + it should successfully scan, extract, and return the contact along with the list of dependencies where it appears. + `, () => { + const highlighted: Contact = { + name: "TJ Holowaychuk" + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedContacts( + [highlighted] + ) + ]); + + const { illuminated } = extractor.extractAndMerge(); + assert.deepEqual( + illuminated, + [ + { name: "TJ Holowaychuk", dependencies: ["express"] } + ] + ); + }); + + it(`Given dependencies where the Highlighted Contact are maintainers of the package, + it should successfully scan, extract, and return the contact along with the list of dependencies where it appears. + `, () => { + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedContacts( + [{ + name: "/.*church/", email: "npm@jonchurch.com" + }, + { email: "c@labsector.com" }] + ) + ]); + + const { illuminated } = extractor.extractAndMerge(); + assert.deepEqual( + illuminated, + [ + { name: "/.*church/", email: "npm@jonchurch.com", dependencies: ["express"] }, + { email: "c@labsector.com", dependencies: ["express"] } + ] + ); + }); +}); + +describe("HighlightedPackages", () => { + it("slould highlight packages that are already well formated", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + foo: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + bar: { + versions: { + "1.2.3": {}, + "1.2.4": {}, + "1.2.5": {} + } + } + } + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + { + foo: ["1.2.3"], + bar: ["1.2.3", "1.2.4"] + } + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 3); + assert.deepEqual(highlightedPackages, ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"]); + }); + + it("slould highlight packages when the package to hightlight is an array of strings", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + foo: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + bar: { + versions: { + "1.2.3": {}, + "1.2.4": {}, + "1.2.5": {} + } + } + } + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 3); + assert.deepEqual(highlightedPackages, ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"]); + }); + + it("slould highlight packages when the package to hightlight is a record of arrays of specs", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + foo: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + bar: { + versions: { + "1.2.3": {}, + "1.2.4": {}, + "1.2.5": {} + } + } + } + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 3); + assert.deepEqual(highlightedPackages, ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"]); + }); + + it("should not highlight a package that is not in the payload", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + foo: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + bar: { + versions: { + "1.2.3": {}, + "1.2.5": {} + } + } + } + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 2); + assert.deepEqual(highlightedPackages, ["foo@1.2.3", "bar@1.2.3"]); + }); + + it("should not crash when there is an invalid spec", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + foo: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + bar: { + versions: { + "1.2.3": {}, + "1.2.5": {} + } + } + } + }; + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + [""] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 0); + }); + + it("should match every versions when there is no versions", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + mocha: { + versions: { + "1.2.3": {}, + "1.2.4": {} + } + }, + jest: { + versions: { + "1.2.1": {}, + "1.2.5": {} + } + } + } + }; + + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["mocha", "jest@1.2.1", "jest"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 4); + assert.deepEqual(highlightedPackages, ["mocha@1.2.3", "mocha@1.2.4", "jest@1.2.1", "jest@1.2.5"]); + }); + + it("should highlight packages with org in their name", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + "@nodesecure/js-x-ray": { + versions: { + "1.0.0": {}, + "1.0.1": {} + } + }, + jest: { + versions: { + "1.2.1": {}, + "1.2.5": {} + } + } + } + }; + + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["@nodesecure/js-x-ray@1.0.0", "@nodesecure/js-x-ray@1.0.1"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 2); + assert.deepEqual(highlightedPackages, ["@nodesecure/js-x-ray@1.0.0", "@nodesecure/js-x-ray@1.0.1"]); + }); + + it("should highlight every package from an org", () => { + const fakePayload: any = { + id: "random-id", + scannerVersion: "1.0.0", + dependencies: { + "@nodesecure/js-x-ray": { + versions: { + "1.0.0": {}, + "1.0.1": {} + } + }, + "@nodesecure/scanner": { + versions: { + "1.2.1": {} + } + }, + foo: { + versions: { + "1.0.0": {} + } + }, + bar: { + versions: { + "1.0.0": {} + } + } + } + }; + + const extractor = new Extractors.Payload(fakePayload, [ + new Extractors.Probes.HighlightedPackages( + ["@nodesecure", "foo@1.0.0"] + ) + ]); + const { highlightedPackages } = extractor.extractAndMerge(); + + assert.strictEqual(highlightedPackages.length, 4); + assert.deepEqual(highlightedPackages, [ + "@nodesecure/js-x-ray@1.0.0", + "@nodesecure/js-x-ray@1.0.1", + "@nodesecure/scanner@1.2.1", + "foo@1.0.0" + ]); + }); +}); + describe("Extractors.Payload events", () => { it("should emits packument and manifest events", () => { const extractor = new Extractors.Payload( diff --git a/workspaces/scanner/test/utils/parseSemverRange.spec.ts b/workspaces/scanner/test/utils/parseSemverRange.spec.ts deleted file mode 100644 index 72d7ff79..00000000 --- a/workspaces/scanner/test/utils/parseSemverRange.spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -// Import Node.js Dependencies -import { describe, it } from "node:test"; -import assert from "node:assert/strict"; - -// Import Internal Dependencies -import { - parseSemverRange -} from "../../src/utils/parseSemverRange.ts"; - -describe("utils.parseSemverRange", () => { - it("should do nothing when the semver ranges are already well formatted", () => { - assert.deepEqual(parseSemverRange({ - foo: "1.2.3", - bar: "1.2.3 || 1.2.4" - }), { - foo: "1.2.3", - bar: "1.2.3 || 1.2.4" - }); - }); - - it("should parse to semver range string when getting an array", () => { - assert.deepEqual(parseSemverRange({ - foo: ["1.2.3"], - bar: ["1.2.3", "1.2.4"] - }), { - foo: "1.2.3", - bar: "1.2.3 || 1.2.4" - }); - }); - - describe("specs", () => { - it("should parse specs to name semver range", () => { - assert.deepEqual(parseSemverRange(["foo@1.2.3", "bar@1.2.3", "bar@1.2.4"]), { - foo: "1.2.3", - bar: "1.2.3 || 1.2.4" - }); - }); - - it("should parse to wildcard when there is no version", () => { - assert.deepEqual(parseSemverRange(["mocha", "jest@1.2.1", "jest"]), { - mocha: "*", - jest: "1.2.1 || *" - }); - }); - - it("should include the org in the name", () => { - assert.deepEqual(parseSemverRange(["@nodesecure/js-x-ray@1.0.0", "@nodesecure/js-x-ray@1.0.1"]), { - "@nodesecure/js-x-ray": "1.0.0 || 1.0.1" - }); - }); - - it("should not parse invalid specs", () => { - assert.deepEqual(parseSemverRange([""]), {}); - }); - - it("should parse scope-only entries as wildcards", () => { - assert.deepEqual(parseSemverRange(["@nodesecure"]), { - "@nodesecure": "*" - }); - }); - - it("should parse scope-only entries alongside regular specs", () => { - assert.deepEqual(parseSemverRange(["@nodesecure", "foo@1.0.0"]), { - "@nodesecure": "*", - foo: "1.0.0" - }); - }); - }); -});