wiki.unix7.org

До этого я никогда не писал код на Go lang. Это первый код, ~300 строк.
Написание этой пары клиент-сервер с обучением заняло ~10 часов.

Больше всего времени занял поиск нужных функций по стандартным библиотекам Go.
Общее впечатление: очень простой и достаточно выразительный язык. После C/С++ немного скучно, но есть много хороших фреймворков и библиотек.

Before that, I had never written code in Go lang. It is my first code, 300 strings
Еhis client-server pair took ~10 hours to write with training.
Most of the time took a search in stangard Go libraries.

For web api I used gin framework.

A bit boring after C/C++.

I designed this code as a distribution with GNU autotools

• https://github.com/kindsoldier/go-microstore

s4srv.yml

#
# $Id$
#
uploadDir: /usr/distfiles
logDir: ./
runDir: ./
accessKey: foo
secretKey: bar
#EOF

config.go

package main
 
import (
    "log"
    "github.com/spf13/viper"
)
 
type AppConfig struct {
    configName string
    uploadDir string
    logDir string
    runDir string
    configDir string
    accessKey string
    secretKey string
}
 
var appConfig = AppConfig{
    configName: "s4srv.yml",
    uploadDir: "/var/upload/microstore",
    logDir: "/var/log/microstore",
    runDir: "/var/run/microstore",
    configDir: "/usr/local/etc/microstore",
    accessKey: "microstore",
    secretKey: "microstore",
}
 
func readConfig(config AppConfig) AppConfig {
    viper.SetDefault("uploadDir", config.uploadDir)
    viper.SetDefault("logDir", config.logDir)
    viper.SetDefault("runDir", config.runDir)
    viper.SetDefault("secretKey", config.secretKey)
    viper.SetDefault("accessKey", config.accessKey)
 
    viper.SetConfigType("yaml")
    viper.SetConfigName(config.configName)
    viper.AddConfigPath(".")
    viper.AddConfigPath(config.configDir)
    err := viper.ReadInConfig()
    if err != nil {
        log.Fatalf("error load config file: %s \n", err)
    }
    config.uploadDir = viper.Get("uploadDir").(string)
    config.logDir = viper.Get("logDir").(string)
    config.runDir = viper.Get("runDir").(string)
    config.accessKey = viper.Get("accessKey").(string)
    config.secretKey = viper.Get("secretKey").(string)
 
    return config
}

main.go

/*
 * Author, Copyright: Oleg Borodin <onborodin@gmail.com>
 */
 
package main
 
import (
    "net/http"
    "fmt"
    "log"
    "path/filepath"
    "os"
    "path"
    "time"
    "io"
    "mime/multipart"
    "regexp"
    "github.com/gin-gonic/gin"
)
 
type StoreFile struct {
    Name string `json:"name"`
    Size int64 `json:"size"`
    ModTime string `json:"modtime"`
}
 
func main() {
    appConfig = readConfig(appConfig)
    ginMain()
}
 
func listDir (dir string, glob string) []StoreFile {
 
    files, err := filepath.Glob(path.Join(dir, glob))
 
    if err != nil {
        log.Fatal(err)
    }
 
    list := []StoreFile{}
    for i := 0; i < len(files); i++ {
        name := files[i]
 
        fi, err := os.Stat(name)
        if err != nil {
            log.Fatal(err)
            continue
        }
        //if fi.IsDir() {
        //    continue
        //}
        if !fi.Mode().IsRegular() {
            continue
        }
        matched, _ :=  regexp.MatchString("^[\\w\\-][\\w\\-\\.]+$", fi.Name())
        if !matched {
            continue
        }
        item := StoreFile{
            Name: fi.Name(),
            Size: fi.Size(),
            ModTime: fi.ModTime().Format(time.RFC3339),
        }
        list = append(list, item)
    }
    return list
}
 
func storeUpload(context *gin.Context) {
    type BindFile struct {
        Name string `form:"name" binding:"required"`
        File *multipart.FileHeader `form:"file" binding:"required"`
    }
 
    type Result struct {
        Name string `json:"name"`
        Error bool `json:"error"`
        Reason string `json:"reason"`
    }
 
    bindFile := BindFile{}
    if err := context.ShouldBind(&bindFile); err != nil {
        res := Result{
            Name: "",
            Error: true,
            Reason: err.Error(),
        }
        context.JSON(http.StatusBadRequest, gin.H{
            "result": res,
        })
        return
    }
 
    file := bindFile.File
    dest := path.Join(appConfig.uploadDir, bindFile.Name)
 
    if err := context.SaveUploadedFile(file, dest); err != nil {
        res := Result{
            Name: "",
            Error: true,
            Reason: err.Error(),
        }
        context.JSON(http.StatusBadRequest, gin.H{
            "result": res,
        })
        return
    }
    res := Result{
        Name: bindFile.Name,
        Error: false,
        Reason: "",
    }
    context.JSON(http.StatusOK, gin.H{
        "result": res,
    })
}
 
func storeDownload(context *gin.Context) {
    //type Req struct {
    //    Name string `form:"name" json:"name"`
    //}
    //var req Req
    //err := context.ShouldBind(&req);
 
    //name := req.Name
    //if name != nil {
    //    name := context.Query("name")
    //}
    //name := context.Query("name")
 
    name := context.Param("name")
    fileName := path.Join(appConfig.uploadDir, name)
 
    if len(name) > 0 && fileIsExists(fileName) {
        context.FileAttachment(fileName, name)
    } else {
        context.Status(http.StatusNotFound)
    }
}
 
func storeHello(context *gin.Context) {
    context.JSON(http.StatusOK, gin.H{
        "result": "hello",
    })
}
 
func storeIndex(context *gin.Context) {
    context.JSON(http.StatusOK, gin.H{
        "result" : listDir(appConfig.uploadDir, "*"),
    })
}
 
func logFormatter() func(param gin.LogFormatterParams) string {
    return func(param gin.LogFormatterParams) string {
        return fmt.Sprintf("%s - [%s] %s %s %s %d %s \"%s\" \"%s\"\n",
            param.ClientIP,
            param.TimeStamp.Format(time.RFC3339),
            param.Method,
            param.Path,
            param.Request.Proto,
            param.StatusCode,
            param.Latency,
            param.Request.UserAgent(),
            param.ErrorMessage,
        )
    }
}
 
func ginMain() {
 
    gin.SetMode(gin.ReleaseMode)
    gin.DisableConsoleColor()
 
    logFile, err := os.Create(path.Join(appConfig.logDir, "access"))
    if err != nil {
      panic(err)
    }
    gin.DefaultWriter = io.MultiWriter(logFile, os.Stdout)
 
    router := gin.New()
    router.Use(gin.LoggerWithFormatter(logFormatter()))
    router.MaxMultipartMemory = 8*1024*1024
 
    accounts := gin.Accounts{
        appConfig.accessKey: appConfig.secretKey,
    }
    authorized := router.Group("/store/api/v1", gin.BasicAuth(accounts))
 
    authorized.GET("/hello", storeHello)
    authorized.GET("/index", storeIndex)
    authorized.POST("/upload", storeUpload)
    authorized.GET("/download/:name", storeDownload)
 
    router.Run(":8089")
}

tools.go

/*
 * Author, Copyright: Oleg Borodin <onborodin@gmail.com>
 */
 
package main
 
import (
    "os"
)
 
func fileIsExists(name string) bool {
    fi, err := os.Stat(name)
    if err != nil {
        if os.IsNotExist(err) {
            return false
        }
    }
    return !fi.IsDir()
}

s4cli.go

package main
 
import (
    "fmt"
    "strings"
    "os"
    "os/user"
    "net/http"
    "io"
    "io/ioutil"
    "encoding/json"
    "net/url"
    "path"
    "path/filepath"
    "errors"
    "mime/multipart"
    "flag"
)
 
type StoreFile struct {
    Name string `json:"name"`
    Size int64  `json:"size"`
    ModTime string `json:"modtime"`
 
}
 
type IndexResult struct {
    Result []StoreFile `json:"result"`
}
 
func urlAppend(orig string, append string) (string, error) {
    u, err := url.Parse(orig)
    if err != nil {
        return orig, err
    }
    u.Path = path.Join(u.Path, append)
    return u.String(), nil
}
 
func expandPath(filePath string) (string, error) {
 
    if strings.HasPrefix(filePath, "~/") {
        usr, err := user.Current()
        if err != nil {
            return filePath, err
        }
        pathArr := strings.Split(filePath, string(os.PathSeparator))
        fmt.Println(pathArr)
        filePath = filepath.Join(usr.HomeDir, string(os.PathSeparator))
        filePath = filepath.Join(filePath, strings.Join(pathArr[1:], string(os.PathSeparator)))
    }
 
    if path.IsAbs(filePath) {
        return filePath, nil
    }
    cwd, err := os.Getwd()
    if err != nil {
        return filePath, err
    }
    return path.Join(cwd, filePath), nil
}
 
func index(baseUrl string) error {
    url, err := urlAppend(baseUrl, "/index")
    if err != nil {
        return err
    }
    resp, err := http.Get(url)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return err
    }
 
    if len(body) > 0 {
        var index IndexResult
        json.Unmarshal([]byte(body), &index)
        len := len(index.Result)
        fmt.Printf("%d:\n", len)
        for i := 0; i < len; i++ {
            fmt.Printf("%s\t%s\t%d\n",
                index.Result[i].Name,
                index.Result[i].ModTime,
                index.Result[i].Size,
            )
        }
    }
    return nil
}
 
func printHeader(header http.Header) {
    for key, val := range header {
        fmt.Printf("%s: %s\n", key, strings.Join(val, " "))
    }
}
 
func download(baseUrl string, name string) error {
 
    url, err := urlAppend(baseUrl, fmt.Sprintf("/download/%s", name))
    if err != nil {
        return err
    }
    resp, err := http.Get(url)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
 
    if resp.StatusCode != http.StatusOK {
        return errors.New(http.StatusText(resp.StatusCode))
    }
 
    out, err := os.Create(name)
    if err != nil {
        return err
    }
    defer out.Close()
    //_, err = io.Copy(out, resp.Body)
    fmt.Println(resp.Status)
 
    printHeader(resp.Header)
 
    buf := make([]byte,  128 * 1024)
    _, err = io.CopyBuffer(out, resp.Body, buf)
    return err
}
 
func upload(baseUrl string, filename string) error {
 
    url, err := urlAppend(baseUrl, "/upload")
    if err != nil {
        return err
    }
 
    pipeOut, pipeIn := io.Pipe()
    writer := multipart.NewWriter(pipeIn)
    go func() {
        defer pipeIn.Close()
        defer writer.Close()
 
        err = writer.WriteField("name", filepath.Base(filename))
        part, err := writer.CreateFormFile("file", filepath.Base(filename))
        if err != nil {
            return
        }
        file, err := os.Open(filename)
        if err != nil {
            return
        }
        defer file.Close()
        if _, err = io.Copy(part, file); err != nil {
            return
        }
    }()
 
    resp, err := http.Post(url, writer.FormDataContentType(), pipeOut)
    resp_body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return err
    }
    fmt.Println(resp.Status)
    printHeader(resp.Header)
    fmt.Println(string(resp_body))
 
    return err
}
 
func main() {
 
    optNode := flag.String("node", "http://foo:bar@localhost:8089", "node")
 
    exeName := filepath.Base(os.Args[0])
    flag.Usage = func() {
        fmt.Printf("usage: %s [global option] command [command option]\n", exeName)
        fmt.Println("")
        fmt.Println("commands: list, upload, download")
        fmt.Println("")
        fmt.Println("global option:")
        fmt.Println("")
        flag.PrintDefaults()
        os.Exit(0)
    }
 
    flag.Parse()
 
    apiPath := "/store/api/v1/"
    apiUrl, err := urlAppend(*optNode, apiPath)
 
    localArgs := flag.Args()
    var command string
 
    if len(localArgs) == 0 {
        flag.Usage()
    }
 
    command = localArgs[0]
    localArgs = localArgs[1:]
 
    if strings.HasPrefix(command, "li") {
        //localCommands := flag.NewFlagSet("index", flag.ExitOnError)
        //localCommands.Parse(localArgs)
 
        err = index(apiUrl)
    } else if strings.HasPrefix(command, "up") {
        localCommands := flag.NewFlagSet("upload", flag.ExitOnError)
        optFileName := localCommands.String("file", "", "file for upload")
        localCommands.Parse(localArgs)
 
        err = upload(apiUrl, *optFileName)
    } else if strings.HasPrefix(command, "down") {
        localCommands := flag.NewFlagSet("download", flag.ExitOnError)
        optFileName := localCommands.String("file", "", "file for download")
        localCommands.Parse(localArgs)
 
        err = download(apiUrl, filepath.Base(*optFileName))
    }
 
    if err != nil {
        fmt.Printf("error: %s\n", err)
    }
 
}

$ ./s4srv

$ ps ax | grep s4srv
37693  -  Is     0:00.32  (s4srv)
$ cat pid
37693

$ ./s4cli -h
Usage: s4cli [-dhlu] [-e value] [-f value] [parameters ...]
 -d, --down        download file
 -e, --node=value  server node [http://foo:bar@localhost:8089]
 -f, --file=value  file name
 -h, --help        help usage
 -l, --list        list files
 -u, --upload      download file

$ ./s4cli --list | tail -3
zlib-1.2.8.tar.xz	2019-03-29T21:34:02+02:00	450776
zoulasc-racoon2-20181215-5c4af73_GH0.tar.gz	2019-09-17T12:33:48+02:00	1365031
zziplib-0.13.62.tar.bz2	2019-03-29T21:34:02+02:00	685770

$ cat access 
127.0.0.1 - [2019-11-25T10:10:58+02:00] GET /store/api/v1/index HTTP/1.1 200 82.565235ms "Go-http-client/1.1" ""
127.0.0.1 - [2019-11-25T10:11:04+02:00] GET /store/api/v1/index HTTP/1.1 200 79.544276ms "Go-http-client/1.1" ""
127.0.0.1 - [2019-11-25T10:11:18+02:00] GET /store/api/v1/index HTTP/1.1 200 80.287835ms "Go-http-client/1.1" ""

$ ./s4cli --upload --file=Makefile.in
200 OK
{"result":{"name":"Makefile.in","error":false,"reason":""}}

$ ./s4cli --list | grep Makefile.in
Makefile.in	2019-11-25T10:14:31+02:00	23161

$ time ./s4cli --upload --file=~/big_blob.bin 
200 OK
{"result":{"name":"big_blob.bin","error":false,"reason":""}}
real	0m15.221s
user	0m1.344s
sys	0m3.417s

$ time ./s4cli --down --file=big_blob.bin
real	0m3.582s
user	0m0.276s
sys	0m2.776s


$ ./s4cli --list | grep big_blob.bin
big_blob.bin	2019-11-25T10:16:27+02:00	1568230702

$ ls -lh big_blob.bin
-rw-r--r--  1 ziggi  wheel   1.5G Nov 25 10:18 big_blob.bin