Ranked packages for python guess

internal/api/types.go

@@ -328,7 +328,7 @@ type LanguageBackend struct {
 	// (which is now wrong).
 	//
 	// This field is mandatory.
-	Guess func(ctx context.Context) (map[PkgName]bool, bool)
+	Guess func(ctx context.Context) (map[string][]PkgName, bool)
 
 	// Installs system dependencies into replit.nix for supported
 	// languages.

internal/backends/dart/dart.go

@@ -293,7 +293,7 @@ func dartRemove(ctx context.Context, pkgs map[api.PkgName]bool) {
 }
 
 // dartGuess stub.
-func dartGuess(context.Context) (map[api.PkgName]bool, bool) {
+func dartGuess(context.Context) (map[string][]api.PkgName, bool) {
 	util.Die("Guess not implemented!")
 
 	return nil, false

internal/backends/elisp/elisp.go

@@ -197,7 +197,7 @@ var ElispBackend = api.LanguageBackend{
 	GuessRegexps: util.Regexps([]string{
 		`\(\s*require\s*'\s*([^)[:space:]]+)[^)]*\)`,
 	}),
-	Guess: func(ctx context.Context) (map[api.PkgName]bool, bool) {
+	Guess: func(ctx context.Context) (map[string][]api.PkgName, bool) {
 		//nolint:ineffassign,wastedassign,staticcheck
 		span, ctx := tracer.StartSpanFromContext(ctx, "elisp guess")
 		defer span.Finish()
@@ -210,7 +210,7 @@ var ElispBackend = api.LanguageBackend{
 		}
 
 		if len(required) == 0 {
-			return map[api.PkgName]bool{}, true
+			return map[string][]api.PkgName{}, true
 		}
 
 		r = regexp.MustCompile(
@@ -242,7 +242,7 @@ var ElispBackend = api.LanguageBackend{
 			clauses = append(clauses, fmt.Sprintf("feature = '%s'", feature))
 		}
 		if len(clauses) == 0 {
-			return map[api.PkgName]bool{}, true
+			return map[string][]api.PkgName{}, true
 		}
 		where := strings.Join(clauses, " OR ")
 		query := fmt.Sprintf("SELECT package FROM provided PR WHERE (%s) "+
@@ -255,9 +255,10 @@ var ElispBackend = api.LanguageBackend{
 		output := string(util.GetCmdOutput([]string{"sqlite3", epkgs, query}))
 
 		r = regexp.MustCompile(`"(.+?)"`)
-		names := map[api.PkgName]bool{}
+		names := map[string][]api.PkgName{}
 		for _, match := range r.FindAllStringSubmatch(output, -1) {
-			names[api.PkgName(match[1])] = true
+			name := match[1]
+			names[name] = []api.PkgName{api.PkgName(name)}
 		}
 		return names, true
 	},

internal/backends/nodejs/grab.go

@@ -57,7 +57,7 @@ var internalModules = []string{
 }
 
 // nodejsGuess implements Guess for nodejs-yarn, nodejs-pnpm and nodejs-npm.
-func nodejsGuess(ctx context.Context) (map[api.PkgName]bool, bool) {
+func nodejsGuess(ctx context.Context) (map[string][]api.PkgName, bool) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "nodejsGuess")
 	defer span.Finish()
 	cwd, err := os.Getwd()
@@ -122,18 +122,18 @@ func findImports(ctx context.Context, dir string) (map[string]bool, error) {
 	return foundImportPaths, nil
 }
 
-func filterImports(ctx context.Context, foundPaths map[string]bool) map[api.PkgName]bool {
+func filterImports(ctx context.Context, foundPaths map[string]bool) map[string][]api.PkgName {
 	//nolint:ineffassign,wastedassign,staticcheck
 	span, ctx := tracer.StartSpanFromContext(ctx, "nodejs.grab.filterImports")
 	defer span.Finish()
-	pkgs := map[api.PkgName]bool{}
+	pkgs := map[string][]api.PkgName{}
 
 	for mod := range foundPaths {
 		if mod == "" {
 			continue
 		}
 
-		if pkgs[api.PkgName(mod)] {
+		if _, ok := pkgs[mod]; ok {
 			continue
 		}
 
@@ -195,7 +195,7 @@ func filterImports(ctx context.Context, foundPaths map[string]bool) map[api.PkgN
 			}
 		}
 
-		pkgs[api.PkgName(mod)] = true
+		pkgs[mod] = []api.PkgName{api.PkgName(mod)}
 	}
 
 	return pkgs

internal/backends/nodejs/nodejs_test.go

@@ -12,7 +12,7 @@ type TestCase struct {
 	scenario    string
 	backend     api.LanguageBackend
 	fileContent string
-	expected    map[api.PkgName]bool
+	expected    map[string]bool
 }
 
 func TestNodejsYarnBackend_Guess(t *testing.T) {
@@ -25,7 +25,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 
 			export const App = () => null;
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"react": true,
 			},
 		},
@@ -35,7 +35,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 			fileContent: `
 			const request = require('request');
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"request": true,
 			},
 		},
@@ -45,15 +45,15 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 			fileContent: `
 			const http = require('http');
 		`,
-			expected: map[api.PkgName]bool{},
+			expected: map[string]bool{},
 		},
 		{
 			scenario: "Ignore internal submodules imports",
 			backend:  NodejsNPMBackend,
 			fileContent: `
 			const dns = require('dns/promises');
 		`,
-			expected: map[api.PkgName]bool{},
+			expected: map[string]bool{},
 		},
 		{
 			scenario: "Returns both requires and imports in mixed file",
@@ -62,7 +62,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 			const request = require('request');
 			import yargs from 'yargs';
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"request": true,
 				"yargs":   true,
 			},
@@ -76,7 +76,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 
 			export const App = () => React.createElement(SomeComponent, {});
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"react": true,
 			},
 		},
@@ -89,7 +89,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 
 			export const App = () => React.createElement(SomeComponent, {});
 		`,
-			expected: map[api.PkgName]bool{},
+			expected: map[string]bool{},
 		},
 		{
 			scenario: "Will process packages in namespace",
@@ -100,7 +100,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 
 			export const App = () => React.createElement(Button, { color: "primary" }, "Hello, World!");
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"react":             true,
 				"@material-ui/core": true,
 			},
@@ -113,7 +113,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 				require("node-fetch");
 			}
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"node-fetch": true,
 			},
 		},
@@ -127,7 +127,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 				require("node-fetch");
 			}
 		`,
-			expected: map[api.PkgName]bool{},
+			expected: map[string]bool{},
 		},
 		{
 			scenario: "dynamic import",
@@ -138,7 +138,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 				import("node-fetch");
 			}
 		`,
-			expected: map[api.PkgName]bool{
+			expected: map[string]bool{
 				"node-fetch": true,
 			},
 		},
@@ -152,7 +152,7 @@ func TestNodejsYarnBackend_Guess(t *testing.T) {
 				import("node-fetch");
 			}
 		`,
-			expected: map[api.PkgName]bool{},
+			expected: map[string]bool{},
 		},
 	}
 
@@ -197,7 +197,7 @@ func verify(t *testing.T, tc TestCase, extension string) {
 	}
 
 	for key := range tc.expected {
-		if !result[key] {
+		if _, ok := result[key]; !ok {
 			t.Errorf("Key %s not found in result map", key)
 		}
 	}

internal/backends/php/php.go

@@ -280,7 +280,7 @@ var PhpComposerBackend = api.LanguageBackend{
 	},
 	ListSpecfile: listSpecfile,
 	ListLockfile: listLockfile,
-	Guess: func(context.Context) (map[api.PkgName]bool, bool) {
+	Guess: func(context.Context) (map[string][]api.PkgName, bool) {
 		util.NotImplemented()
 		return nil, false
 	},

internal/backends/python/grab.go

@@ -244,7 +244,7 @@ var internalModules = map[string]bool{
 	"zoneinfo":        true,
 }
 
-func guess(ctx context.Context, python string) (map[api.PkgName]bool, bool) {
+func guess(ctx context.Context, python string) (map[string][]api.PkgName, bool) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "python.grab.guess")
 	defer span.Finish()
 	cwd, err := os.Getwd()
@@ -278,7 +278,7 @@ func findImports(ctx context.Context, dir string) (map[string]bool, error) {
 	return foundImportPaths, nil
 }
 
-func filterImports(ctx context.Context, foundPkgs map[string]bool) (map[api.PkgName]bool, bool) {
+func filterImports(ctx context.Context, foundPkgs map[string]bool) (map[string][]api.PkgName, bool) {
 	//nolint:ineffassign,wastedassign,staticcheck
 	span, ctx := tracer.StartSpanFromContext(ctx, "python.grab.filterImports")
 	defer span.Finish()
@@ -296,19 +296,26 @@ func filterImports(ctx context.Context, foundPkgs map[string]bool) (map[api.PkgN
 	}
 	defer pypiMap.Close()
 
-	pkgs := map[api.PkgName]bool{}
+	pkgs := map[string][]api.PkgName{}
 
 	for fullModname := range foundPkgs {
 		// try and look it up in Pypi
 		var pkg string
+		var overrides []string
 		var ok bool
 
-		modNameParts := strings.Split(fullModname, ".")
+		modNameParts := strings.Split(strings.ToLower(fullModname), ".")
 		for len(modNameParts) > 0 {
 			testModName := strings.Join(modNameParts, ".")
 
 			// test overrides
-			pkg, ok = moduleToPypiPackageOverride[testModName]
+			overrides, ok = moduleToPypiPackageOverride[testModName]
+			if ok {
+				break
+			}
+
+			// test aliases
+			pkg, ok = moduleToPypiPackageAliases[testModName]
 			if ok {
 				break
 			}
@@ -324,8 +331,17 @@ func filterImports(ctx context.Context, foundPkgs map[string]bool) (map[api.PkgN
 		}
 
 		if ok {
-			name := api.PkgName(pkg)
-			pkgs[normalizePackageName(name)] = true
+			if pkg != "" {
+				name := api.PkgName(pkg)
+				pkgs[pkg] = []api.PkgName{normalizePackageName(name)}
+			} else {
+				group := []api.PkgName{}
+				for _, name := range overrides {
+					name := api.PkgName(name)
+					group = append(group, normalizePackageName(name))
+				}
+				pkgs[overrides[0]] = group
+			}
 		}
 	}
 

internal/backends/python/pypi_map.override.go

@@ -3,20 +3,24 @@ package python
 /*
 Manual module -> package mapping overrides
 */
-var moduleToPypiPackageOverride = map[string]string{
-	"grpc_status":  "grpcio-status",       // 2nd most popular
-	"nvd3":         "python-nvd3",         // not popular enough 6th in popularity
-	"requirements": "requirements-parser", // popular rlbot depends on it, but doesn't supply requires_dist
-	"base62":       "pybase62",            // it was overridden by base-62 which wins due to name match but is less popular by far
-	"faiss":        "faiss-cpu",           // faiss is offered as precompiled wheels, faiss-cpu and faiss-gpu.
-	"graphics":     "graphics.py",         // this package is popular, but the module doesn't match the package name https://anh.cs.luc.edu/python/hands-on/3.1/handsonHtml/graphics.html#a-graphics-introduction
-	"replit.ai":    "replit-ai",           // Replit's AI package
-	/* Proxy packages
-	 *
-	 * These are packages that provide helpful aliases, but otherwise provide no functionality.
-	 * We should prefer the real version.
-	 */
-	"discord": "discord.py",
-	"bs4":     "beautifulsoup4",
-	"glm":     "PyGLM",
+var moduleToPypiPackageOverride = map[string][]string{
+	"grpc_status":  {"grpcio-status"},       // 2nd most popular
+	"nvd3":         {"python-nvd3"},         // not popular enough 6th in popularity
+	"requirements": {"requirements-parser"}, // popular rlbot depends on it, but doesn't supply requires_dist
+	"base62":       {"pybase62"},            // it was overridden by base-62 which wins due to name match but is less popular by far
+	"faiss":        {"faiss-cpu"},           // faiss is offered as precompiled wheels, faiss-cpu and faiss-gpu.
+	"graphics":     {"graphics.py"},         // this package is popular, but the module doesn't match the package name https://anh.cs.luc.edu/python/hands-on/3.1/handsonHtml/graphics.html#a-graphics-introduction
+	"replit.ai":    {"replit-ai"},           // Replit's AI package
+	"glm":          {"PyGLM", "glm"},        // Both of these packages are valid, but PyGLM is a library, glm is an executable.
+}
+
+/* Proxy packages
+ *
+ * These are packages that provide helpful aliases, but otherwise provide no functionality.
+ * We should prefer the real version.
+ */
+var moduleToPypiPackageAliases = map[string]string{
+	"bs4":      "beautifulsoup4",
+	"discord":  "discord.py",
+	"psycopg2": "psycopg2-binary", // psycopg2 is a source package, psycopg2-binary is the dist wheel
 }

internal/backends/python/python.go

@@ -178,6 +178,11 @@ func add(ctx context.Context, pkgs map[api.PkgName]api.PkgSpec, projectName stri
 	cmd := []string{"poetry", "add"}
 	for name, spec := range pkgs {
 		name := string(name)
+		if found, ok := moduleToPypiPackageAliases[name]; ok {
+			delete(pkgs, api.PkgName(name))
+			name = found
+			pkgs[api.PkgName(name)] = api.PkgSpec(spec)
+		}
 		spec := string(spec)
 
 		// NB: this doesn't work if spec has
@@ -196,7 +201,7 @@ func add(ctx context.Context, pkgs map[api.PkgName]api.PkgSpec, projectName stri
 
 func searchPypi(query string) []api.PkgInfo {
 	if renamed, found := moduleToPypiPackageOverride[query]; found {
-		query = renamed
+		query = renamed[0]
 	}
 	results, err := SearchPypi(query)
 	if err != nil {
@@ -298,7 +303,7 @@ func makePythonPoetryBackend(python string) api.LanguageBackend {
 			return pkgs
 		},
 		GuessRegexps: pythonGuessRegexps,
-		Guess:        func(ctx context.Context) (map[api.PkgName]bool, bool) { return guess(ctx, python) },
+		Guess:        func(ctx context.Context) (map[string][]api.PkgName, bool) { return guess(ctx, python) },
 		InstallReplitNixSystemDependencies: func(ctx context.Context, pkgs []api.PkgName) {
 			//nolint:ineffassign,wastedassign,staticcheck
 			span, ctx := tracer.StartSpanFromContext(ctx, "python.InstallReplitNixSystemDependencies")
@@ -375,6 +380,11 @@ func makePythonPipBackend(python string) api.LanguageBackend {
 			for name, spec := range pkgs {
 				name := string(name)
 				spec := string(spec)
+				if found, ok := moduleToPypiPackageAliases[name]; ok {
+					delete(pkgs, api.PkgName(name))
+					name = found
+					pkgs[api.PkgName(name)] = api.PkgSpec(spec)
+				}
 
 				cmd = append(cmd, name+spec)
 			}
@@ -459,7 +469,7 @@ func makePythonPipBackend(python string) api.LanguageBackend {
 			return pkgs
 		},
 		GuessRegexps: pythonGuessRegexps,
-		Guess:        func(ctx context.Context) (map[api.PkgName]bool, bool) { return guess(ctx, python) },
+		Guess:        func(ctx context.Context) (map[string][]api.PkgName, bool) { return guess(ctx, python) },
 		InstallReplitNixSystemDependencies: func(ctx context.Context, pkgs []api.PkgName) {
 			//nolint:ineffassign,wastedassign,staticcheck
 			span, ctx := tracer.StartSpanFromContext(ctx, "python.InstallReplitNixSystemDependencies")