[WIP] Building a CLI for migrating between storage backends

cmd/migrator/cmd/catalog.go

@@ -0,0 +1,74 @@
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/gomods/athens/pkg/storage"
+)
+
+// returns the info bytes, mod, and zip (in that order)
+func getCompleteMod(
+	ctx context.Context,
+	getter storage.Getter,
+	mod string,
+	ver string,
+) ([]byte, []byte, io.ReadCloser, error) {
+	infoBytes, err := getter.Info(ctx, mod, ver)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	modBytes, err := getter.GoMod(ctx, mod, ver)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	zip, err := getter.Zip(ctx, mod, ver)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	return infoBytes, modBytes, zip, nil
+}
+
+func transfer(
+	ctx context.Context,
+	cataloger storage.Cataloger,
+	from storage.Getter,
+	to storage.Saver,
+) error {
+	token := ""
+	// TODO: parallelize this
+	for {
+		pathParams, newToken, err := cataloger.Catalog(ctx, token, 100)
+		if err != nil {
+			return err
+		}
+		for _, pathParam := range pathParams {
+			mod := pathParam.Module
+			ver := pathParam.Version
+			infoBytes, modBytes, zip, err := getCompleteMod(
+				ctx,
+				from,
+				mod,
+				ver,
+			)
+			if err != nil {
+				return err
+			}
+			saveErr := to.Save(
+				ctx,
+				mod,
+				ver,
+				modBytes,
+				zip,
+				infoBytes,
+			)
+			if saveErr != nil {
+				return err
+			}
+		}
+		if newToken == "" {
+			return nil
+		}
+		token = newToken
+	}
+}

cmd/migrator/cmd/errlog.go

@@ -0,0 +1,11 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+)
+
+func errLog(fmtString string, args ...interface{}) {
+	fmt.Printf(fmtString, args...)
+	os.Exit(1)
+}

cmd/migrator/cmd/migrate.go

@@ -0,0 +1,105 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/gomods/athens/cmd/proxy/actions"
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/spf13/cobra"
+)
+
+type migrateCmd struct {
+	from *string
+	to   *string
+}
+
+func (m *migrateCmd) run(cmd *cobra.Command, args []string) {
+	cfg, err := config.ParseConfigFile(cfgFile)
+	if err != nil {
+		errLog("Couldn't read config file (%s)", err)
+	}
+	// TODO: make this a flag
+	timeout := 1 * time.Second
+	ctx := context.Background()
+
+	fromStorage, err := actions.GetStorage(
+		*m.from,
+		cfg.Storage,
+		timeout,
+	)
+	if err != nil {
+		errLog(
+			"Error getting 'from' storage %s (%s)",
+			*m.from,
+			err,
+		)
+	}
+	toStorage, err := actions.GetStorage(
+		*m.to,
+		cfg.Storage,
+		timeout,
+	)
+	if err != nil {
+		errLog(
+			"Error getting 'to' storage %s (%s)",
+			*m.to,
+			err,
+		)
+	}
+
+	cataloger, ok := fromStorage.(storage.Cataloger)
+	if !ok {
+		errLog(
+			"'from' storage %s doesn't support cataloging, sorry :(",
+			*m.from,
+		)
+	}
+	if err := transfer(
+		ctx,
+		cataloger,
+		fromStorage,
+		toStorage,
+	); err != nil {
+		errLog("Error transfering (%s)", err)
+	}
+
+	fmt.Println("migrate called")
+}
+
+func init() {
+	cmd := &migrateCmd{}
+	cobraCmd := &cobra.Command{
+		Use:   "migrate",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+	and usage of using your command. For example:
+
+	Cobra is a CLI library for Go that empowers applications.
+	This application is a tool to generate the needed files
+	to quickly create a Cobra application.`,
+		Run: cmd.run,
+	}
+	rootCmd.AddCommand(cobraCmd)
+
+	// Here you will define your flags and configuration settings.
+	cmd.from = cobraCmd.PersistentFlags().String(
+		"from",
+		"disk",
+		"The Athens storage backend to migrate from",
+	)
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	cmd.to = cobraCmd.PersistentFlags().String(
+		"to",
+		"s3",
+		"The Athens storage backend to migrate to",
+	)
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// migrateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/migrator/cmd/root.go

@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/spf13/viper"
+)
+
+var cfgFile string
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "migrator",
+	Short: "A brief description of your application",
+	Long: `A longer description that spans multiple lines and likely contains
+examples and usage of using your application. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	//	Run: func(cmd *cobra.Command, args []string) { },
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.migrator.yaml)")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Find home directory.
+		home, err := homedir.Dir()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		// Search config in home directory with name ".migrator" (without extension).
+		viper.AddConfigPath(home)
+		viper.SetConfigName(".migrator")
+	}
+
+	viper.AutomaticEnv() // read in environment variables that match
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	}
+}

cmd/migrator/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/gomods/athens/cmd/migrator/cmd"
+
+func main() {
+	cmd.Execute()
+}

go.mod

@@ -37,7 +37,9 @@ require (
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/afero v1.1.2
+	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3 // indirect
+	github.com/spf13/viper v1.6.3
 	github.com/stretchr/testify v1.4.0
 	github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245
 	github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 // indirect