assets: convert usage of go-bindata to Go embed (#6851)

Co-authored-by: Joe Chen <jc@unknwon.io>

.deepsource.toml

@@ -1,7 +1,5 @@
 version = 1
 
-exclude_patterns = ["**/*_gen.go"]
-
 [[analyzers]]
 name = "docker"
 enabled = true
@@ -23,4 +21,4 @@ enabled = true
 
 [[transformers]]
 name = "gofmt"
-enabled = true
+enabled = true

.github/workflows/go.yml

@@ -32,11 +32,6 @@ jobs:
           args: --timeout=30m
       - name: Install Task
         uses: arduino/setup-task@v1
-      - name: Install go-bindata
-        shell: bash
-        run: |
-          curl --silent --location --output /usr/local/bin/go-bindata https://github.com/kevinburke/go-bindata/releases/download/v3.23.0/go-bindata-linux-amd64
-          chmod +x /usr/local/bin/go-bindata
       - name: Check Go module tidiness and generated files
         shell: bash
         run: |

Makefile

@@ -37,11 +37,6 @@ pack:
 
 release: build pack
 
-generate: clean
-	go generate internal/assets/conf/conf.go
-	go generate internal/assets/templates/templates.go
-	go generate internal/assets/public/public.go
-
 less: clean public/css/gogs.min.css
 
 public/css/gogs.min.css: $(LESS_FILES)

Taskfile.yml

@@ -30,14 +30,9 @@ tasks:
     sources:
       - gogs.go
       - internal/**/*.go
-
-  generate-bindata:
-    desc: Generate bindata for all assets.
-    deps: [clean]
-    cmds:
-      - go generate internal/assets/conf/conf.go
-      - go generate internal/assets/templates/templates.go
-      - go generate internal/assets/public/public.go
+      - conf/**
+      - public/**
+      - templates/**
 
   generate-schemadoc:
     desc: Generate database schema documentation.
@@ -46,7 +41,7 @@ tasks:
 
   generate:
     desc: Run all go:generate commands.
-    deps: [generate-bindata, generate-schemadoc]
+    deps: [generate-schemadoc]
 
   test:
     desc: Run all tests.

conf/embed.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conf
+
+import (
+	"embed"
+)
+
+//go:embed app.ini **/*
+var Files embed.FS
+
+// FileNames returns a list of filenames exists in the given direction within
+// Files. The list includes names of subdirectories.
+func FileNames(dir string) ([]string, error) {
+	entries, err := Files.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	fileNames := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		fileNames = append(fileNames, entry.Name())
+	}
+	return fileNames, nil
+}

conf/embed_test.go

@@ -0,0 +1,20 @@
+// Copyright 2022 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conf
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileNames(t *testing.T) {
+	names, err := FileNames(".")
+	require.NoError(t, err)
+
+	want := []string{"app.ini", "auth.d", "gitignore", "label", "license", "locale", "readme"}
+	assert.Equal(t, want, names)
+}

internal/assets/assets.go

@@ -1,13 +0,0 @@
-package assets
-
-import (
-	"strings"
-)
-
-// IsErrNotFound returns true if the error is asset not found.
-func IsErrNotFound(err error) bool {
-	if err == nil {
-		return false
-	}
-	return strings.Contains(err.Error(), "not found")
-}

internal/assets/conf/conf.go

@@ -1,7 +0,0 @@
-// Copyright 2020 The Gogs Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package conf
-
-//go:generate go-bindata -nomemcopy -nometadata -pkg=conf -ignore="\\.DS_Store|README.md|TRANSLATORS|auth.d" -prefix=../../../ -debug=false -o=conf_gen.go ../../../conf/...

internal/assets/public/public.go

@@ -1,143 +0,0 @@
-// Copyright 2020 The Gogs Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package public
-
-import (
-	"bytes"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"gogs.io/gogs/internal/assets"
-)
-
-//go:generate go-bindata -nomemcopy -nometadata -pkg=public -ignore="\\.DS_Store|less" -prefix=../../../public -debug=false -o=public_gen.go ../../../public/...
-
-/*
-	This file is a modified version of https://github.com/go-bindata/go-bindata/pull/18.
-*/
-
-type fileInfo struct {
-	name string
-	size int64
-}
-
-func (d fileInfo) Name() string {
-	return d.name
-}
-
-func (d fileInfo) Size() int64 {
-	return d.size
-}
-
-func (fileInfo) Mode() os.FileMode {
-	return os.FileMode(0644) | os.ModeDir
-}
-
-func (fileInfo) ModTime() time.Time {
-	return time.Time{}
-}
-
-// IsDir return file whether a directory
-func (*fileInfo) IsDir() bool {
-	return true
-}
-
-func (fileInfo) Sys() interface{} {
-	return nil
-}
-
-// file implements the http.File interface.
-type file struct {
-	name string
-	*bytes.Reader
-
-	children       []os.FileInfo
-	childrenOffset int
-}
-
-func (*file) Close() error {
-	return nil
-}
-
-// ⚠️ WARNING: This method is not concurrent-safe.
-func (f *file) Readdir(count int) ([]os.FileInfo, error) {
-	if len(f.children) == 0 {
-		return nil, os.ErrNotExist
-	}
-
-	if count <= 0 {
-		return f.children, nil
-	}
-
-	if f.childrenOffset+count > len(f.children) {
-		count = len(f.children) - f.childrenOffset
-	}
-	offset := f.childrenOffset
-	f.childrenOffset += count
-	return f.children[offset : offset+count], nil
-}
-
-func (f *file) Stat() (os.FileInfo, error) {
-	childCount := len(f.children)
-	if childCount != 0 {
-		return &fileInfo{
-			name: f.name,
-			size: int64(childCount),
-		}, nil
-	}
-	return AssetInfo(f.name)
-}
-
-// fileSystem implements the http.FileSystem interface.
-type fileSystem struct{}
-
-func (*fileSystem) Open(name string) (http.File, error) {
-	if len(name) > 0 && name[0] == '/' {
-		name = name[1:]
-	}
-
-	// Attempt to get it as a file
-	p, err := Asset(name)
-	if err != nil && !assets.IsErrNotFound(err) {
-		return nil, err
-	} else if err == nil {
-		return &file{
-			name:   name,
-			Reader: bytes.NewReader(p),
-		}, nil
-	}
-
-	// Attempt to get it as a directory
-	paths, err := AssetDir(name)
-	if err != nil && !assets.IsErrNotFound(err) {
-		return nil, err
-	}
-
-	infos := make([]os.FileInfo, len(paths))
-	for i, path := range paths {
-		path = filepath.Join(name, path)
-		info, err := AssetInfo(path)
-		if err != nil {
-			if !assets.IsErrNotFound(err) {
-				return nil, err
-			}
-			// Not found as a file, assume it's a directory.
-			infos[i] = &fileInfo{name: path}
-		} else {
-			infos[i] = info
-		}
-	}
-	return &file{
-		name:     name,
-		children: infos,
-	}, nil
-}
-
-// NewFileSystem returns an http.FileSystem instance backed by embedded assets.
-func NewFileSystem() http.FileSystem {
-	return &fileSystem{}
-}

internal/cmd/web.go

@@ -29,9 +29,8 @@ import (
 	"gopkg.in/macaron.v1"
 	log "unknwon.dev/clog/v2"
 
+	embedConf "gogs.io/gogs/conf"
 	"gogs.io/gogs/internal/app"
-	"gogs.io/gogs/internal/assets/public"
-	"gogs.io/gogs/internal/assets/templates"
 	"gogs.io/gogs/internal/conf"
 	"gogs.io/gogs/internal/context"
 	"gogs.io/gogs/internal/db"
@@ -46,6 +45,8 @@ import (
 	"gogs.io/gogs/internal/route/repo"
 	"gogs.io/gogs/internal/route/user"
 	"gogs.io/gogs/internal/template"
+	"gogs.io/gogs/public"
+	"gogs.io/gogs/templates"
 )
 
 var Web = cli.Command{
@@ -83,7 +84,7 @@ func newMacaron() *macaron.Macaron {
 	))
 	var publicFs http.FileSystem
 	if !conf.Server.LoadAssetsFromDisk {
-		publicFs = public.NewFileSystem()
+		publicFs = http.FS(public.Files)
 	}
 	m.Use(macaron.Static(
 		filepath.Join(conf.WorkDir(), "public"),
@@ -119,13 +120,16 @@ func newMacaron() *macaron.Macaron {
 	}
 	m.Use(macaron.Renderer(renderOpt))
 
-	localeNames, err := conf.AssetDir("conf/locale")
+	localeNames, err := embedConf.FileNames("locale")
 	if err != nil {
 		log.Fatal("Failed to list locale files: %v", err)
 	}
 	localeFiles := make(map[string][]byte)
 	for _, name := range localeNames {
-		localeFiles[name] = conf.MustAsset("conf/locale/" + name)
+		localeFiles[name], err = embedConf.Files.ReadFile("locale/" + name)
+		if err != nil {
+			log.Fatal("Failed to read locale file %q: %v", name, err)
+		}
 	}
 	m.Use(i18n.I18n(i18n.Options{
 		SubURL:          conf.Server.Subpath,

internal/conf/conf.go

@@ -22,7 +22,7 @@ import (
 	"gopkg.in/ini.v1"
 	log "unknwon.dev/clog/v2"
 
-	"gogs.io/gogs/internal/assets/conf"
+	"gogs.io/gogs/conf"
 	"gogs.io/gogs/internal/osutil"
 	"gogs.io/gogs/internal/semverutil"
 )
@@ -35,21 +35,6 @@ func init() {
 	}
 }
 
-// Asset is a wrapper for getting conf assets.
-func Asset(name string) ([]byte, error) {
-	return conf.Asset(name)
-}
-
-// AssetDir is a wrapper for getting conf assets.
-func AssetDir(name string) ([]string, error) {
-	return conf.AssetDir(name)
-}
-
-// MustAsset is a wrapper for getting conf assets.
-func MustAsset(name string) []byte {
-	return conf.MustAsset(name)
-}
-
 // File is the configuration object.
 var File *ini.File
 
@@ -62,12 +47,16 @@ var File *ini.File
 //
 // ⚠️ WARNING: Do not print anything in this function other than warnings.
 func Init(customConf string) error {
-	var err error
+	data, err := conf.Files.ReadFile("app.ini")
+	if err != nil {
+		return errors.Wrap(err, `read default "app.ini"`)
+	}
+
 	File, err = ini.LoadSources(ini.LoadOptions{
 		IgnoreInlineComment: true,
-	}, conf.MustAsset("conf/app.ini"))
+	}, data)
 	if err != nil {
-		return errors.Wrap(err, "parse 'conf/app.ini'")
+		return errors.Wrap(err, `parse "app.ini"`)
 	}
 	File.NameMapper = ini.SnackCase
 

internal/conf/conf_test.go

@@ -15,27 +15,6 @@ import (
 	"gogs.io/gogs/internal/testutil"
 )
 
-func TestAsset(t *testing.T) {
-	// Make sure it does not blow up
-	_, err := Asset("conf/app.ini")
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestAssetDir(t *testing.T) {
-	// Make sure it does not blow up
-	_, err := AssetDir("conf")
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestMustAsset(_ *testing.T) {
-	// Make sure it does not blow up
-	MustAsset("conf/app.ini")
-}
-
 func TestInit(t *testing.T) {
 	ini.PrettyFormat = false
 	defer func() {

internal/db/repo.go

@@ -29,6 +29,7 @@ import (
 	"github.com/gogs/git-module"
 	api "github.com/gogs/go-gogs-client"
 
+	embedConf "gogs.io/gogs/conf"
 	"gogs.io/gogs/internal/avatar"
 	"gogs.io/gogs/internal/conf"
 	"gogs.io/gogs/internal/db/errors"
@@ -57,10 +58,11 @@ func LoadRepoConfig() {
 	types := []string{"gitignore", "license", "readme", "label"}
 	typeFiles := make([][]string, 4)
 	for i, t := range types {
-		files, err := conf.AssetDir("conf/" + t)
+		files, err := embedConf.FileNames(t)
 		if err != nil {
-			log.Fatal("Failed to get %s files: %v", t, err)
+			log.Fatal("Failed to get %q files: %v", t, err)
 		}
+
 		customPath := filepath.Join(conf.CustomDir(), "conf", t)
 		if com.IsDir(customPath) {
 			customFiles, err := com.StatDir(customPath)
@@ -940,14 +942,14 @@ type CreateRepoOptions struct {
 }
 
 func getRepoInitFile(tp, name string) ([]byte, error) {
-	relPath := path.Join("conf", tp, strings.TrimLeft(path.Clean("/"+name), "/"))
+	relPath := path.Join(tp, strings.TrimLeft(path.Clean("/"+name), "/"))
 
 	// Use custom file when available.
-	customPath := filepath.Join(conf.CustomDir(), relPath)
+	customPath := filepath.Join(conf.CustomDir(), "conf", relPath)
 	if osutil.IsFile(customPath) {
 		return ioutil.ReadFile(customPath)
 	}
-	return conf.Asset(relPath)
+	return embedConf.Files.ReadFile(relPath)
 }
 
 func prepareRepoCommit(repo *Repository, tmpDir, repoPath string, opts CreateRepoOptions) error {

internal/email/email.go

@@ -15,9 +15,9 @@ import (
 	"gopkg.in/macaron.v1"
 	log "unknwon.dev/clog/v2"
 
-	"gogs.io/gogs/internal/assets/templates"
 	"gogs.io/gogs/internal/conf"
 	"gogs.io/gogs/internal/markup"
+	"gogs.io/gogs/templates"
 )
 
 const (

public/embed.go

@@ -0,0 +1,12 @@
+// Copyright 2022 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package public
+
+import (
+	"embed"
+)
+
+//go:embed assets/* css/* img/* js/* plugins/*
+var Files embed.FS

internal/assets/templates/templates.go → templates/embed.go

@@ -6,8 +6,10 @@ package templates
 
 import (
 	"bytes"
+	"embed"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"path"
 	"strings"
@@ -17,7 +19,8 @@ import (
 	"gogs.io/gogs/internal/osutil"
 )
 
-//go:generate go-bindata -nomemcopy -nometadata -ignore="\\.DS_Store" -pkg=templates -prefix=../../../templates -debug=false -o=templates_gen.go ../../../templates/...
+//go:embed *.tmpl **/*
+var files embed.FS
 
 // fileSystem implements the macaron.TemplateFileSystem interface.
 type fileSystem struct {
@@ -37,6 +40,20 @@ func (fs *fileSystem) Get(name string) (io.Reader, error) {
 	return nil, fmt.Errorf("file %q not found", name)
 }
 
+func mustNames(fsys fs.FS) []string {
+	var names []string
+	walkDirFunc := func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
+			names = append(names, path)
+		}
+		return nil
+	}
+	if err := fs.WalkDir(fsys, ".", walkDirFunc); err != nil {
+		panic("assetNames failure: " + err.Error())
+	}
+	return names
+}
+
 // NewTemplateFileSystem returns a macaron.TemplateFileSystem instance for embedded assets.
 // The argument "dir" can be used to serve subset of embedded assets. Template file
 // found under the "customDir" on disk has higher precedence over embedded assets.
@@ -45,30 +62,28 @@ func NewTemplateFileSystem(dir, customDir string) macaron.TemplateFileSystem {
 		dir += "/"
 	}
 
-	var files []macaron.TemplateFile
-	names := AssetNames()
+	var err error
+	var tmplFiles []macaron.TemplateFile
+	names := mustNames(files)
 	for _, name := range names {
 		if !strings.HasPrefix(name, dir) {
 			continue
 		}
-
 		// Check if corresponding custom file exists
-		var err error
 		var data []byte
 		fpath := path.Join(customDir, name)
 		if osutil.IsFile(fpath) {
 			data, err = ioutil.ReadFile(fpath)
 		} else {
-			data, err = Asset(name)
+			data, err = files.ReadFile(name)
 		}
 		if err != nil {
 			panic(err)
 		}
-
 		name = strings.TrimPrefix(name, dir)
 		ext := path.Ext(name)
 		name = strings.TrimSuffix(name, ext)
-		files = append(files, macaron.NewTplFile(name, data, ext))
+		tmplFiles = append(tmplFiles, macaron.NewTplFile(name, data, ext))
 	}
-	return &fileSystem{files: files}
+	return &fileSystem{files: tmplFiles}
 }