change git

attenuator.go

@@ -0,0 +1,59 @@
+package goutil
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Attenuator struct {
+	sync.Mutex
+	memory *Memory
+}
+
+func NewAttenuator() *Attenuator {
+	return &Attenuator{
+		memory: NewMemory(),
+	}
+}
+
+func (a *Attenuator) Do(key string, fun func()) bool {
+	a.Lock()
+	defer a.Unlock()
+
+	val := a.memory.Get(key)
+	if val == nil {
+		fun()
+		a.memory.Set(key, fmt.Sprintf("%d,1", time.Now().Unix()), time.Hour*time.Duration(24*365*100))
+		return true
+	}
+
+	fields := strings.Split(val.(string), ",")
+
+	timestamp, _ := strconv.ParseInt(fields[0], 10, 64)
+	count, _ := strconv.ParseInt(fields[0], 10, 64)
+
+	count += 1
+	nowTime := time.Now().Unix()
+	if nowTime-timestamp <= int64(math.Pow(2, float64(count)))*60 {
+		return false
+	}
+
+	fun()
+	a.memory.Set(key, fmt.Sprintf("%d,%d", nowTime, count), time.Hour*time.Duration(24*365*100))
+	return true
+}
+
+func (a *Attenuator) Clear(key string) bool {
+	a.Lock()
+	defer a.Unlock()
+
+	if a.memory.IsExist(key) {
+		a.memory.Delete(key)
+		return true
+	}
+	return false
+}

download.go

@@ -0,0 +1,208 @@
+package goutil
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type progressWriter struct {
+	onProgress  func(totalSize int64, processSize int64)
+	totalSize   atomic.Int64
+	processSize atomic.Int64
+	lock        sync.Mutex
+}
+
+func (p *progressWriter) Reset() {
+	p.totalSize.Store(0)
+	p.processSize.Store(0)
+}
+func (p *progressWriter) SetTotal(totalSize int64) {
+	p.totalSize.Store(totalSize)
+}
+func (p *progressWriter) Add(size int64) {
+	p.processSize.Add(size)
+}
+func (p *progressWriter) Write(dat []byte) (n int, err error) {
+	n = len(dat)
+	p.processSize.Add(int64(n))
+	if p.onProgress != nil {
+		p.onProgress(p.totalSize.Load(), p.processSize.Load())
+	}
+	return
+}
+
+type Downloader struct {
+	concurrency int
+	resume      bool
+	progress    *progressWriter
+}
+
+func NewDownloader(concurrency int, resume bool) *Downloader {
+	return &Downloader{concurrency: concurrency, resume: resume}
+}
+
+func (d *Downloader) Download(strURL, filename string, onProgress func(totalSize int64, processSize int64)) error {
+
+	resp, err := http.Head(strURL)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 300 && resp.StatusCode <= 399 {
+		strURL = resp.Header.Get("Location")
+		resp, err = http.Head(strURL)
+		if err != nil {
+			return err
+		}
+	}
+	if filename == "" {
+		u, _ := url.Parse(strURL)
+		filename = filepath.Base(u.Path)
+	}
+	if filename == "" {
+		contentType := resp.Header.Get("Accept-Ranges")
+		panic(errors.New(contentType)) //todo
+	}
+
+	d.progress = &progressWriter{onProgress: onProgress}
+	if d.concurrency > 1 && resp.StatusCode == http.StatusOK && resp.Header.Get("Accept-Ranges") == "bytes" {
+		return d.multiDownload(strURL, filename, int(resp.ContentLength))
+	}
+
+	return d.singleDownload(strURL, filename)
+}
+
+func (d *Downloader) multiDownload(strURL, filename string, contentLen int) error {
+	d.progress.SetTotal(int64(contentLen))
+	tempFilename := fmt.Sprintf("%d", time.Now().Unix())
+	partSize := contentLen / d.concurrency
+
+	var wg sync.WaitGroup
+	wg.Add(d.concurrency)
+
+	rangeStart := 0
+
+	for i := 0; i < d.concurrency; i++ {
+		go func(i, rangeStart int) {
+			defer wg.Done()
+
+			rangeEnd := rangeStart + partSize
+			// 最后一部分，总长度不能超过 ContentLength
+			if i == d.concurrency-1 {
+				rangeEnd = contentLen
+			}
+
+			downloaded := 0
+			partFileName := d.getPartFilename(tempFilename, i)
+			if d.resume {
+				content, err := os.ReadFile(partFileName)
+				if err == nil {
+					downloaded = len(content)
+				}
+				d.progress.Add(int64(downloaded))
+			}
+
+			d.downloadPartial(strURL, partFileName, rangeStart+downloaded, rangeEnd, i)
+
+		}(i, rangeStart)
+
+		rangeStart += partSize + 1
+	}
+
+	wg.Wait()
+
+	d.merge(filename, tempFilename)
+
+	return nil
+}
+
+func (d *Downloader) downloadPartial(strURL, partFilename string, rangeStart, rangeEnd, i int) {
+	if rangeStart >= rangeEnd {
+		return
+	}
+
+	req, err := http.NewRequest("GET", strURL, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd))
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	flags := os.O_CREATE | os.O_WRONLY
+	if d.resume {
+		flags |= os.O_APPEND
+	}
+
+	partFile, err := os.OpenFile(partFilename, flags, 0666)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer partFile.Close()
+
+	buf := make([]byte, 32*1024)
+	_, err = io.CopyBuffer(io.MultiWriter(partFile, d.progress), resp.Body, buf)
+	if err != nil {
+		if err == io.EOF {
+			return
+		}
+		log.Fatal(err)
+	}
+}
+
+func (d *Downloader) merge(filename, tempFilename string) error {
+	destFile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	for i := 0; i < d.concurrency; i++ {
+		partFileName := d.getPartFilename(tempFilename, i)
+		partFile, err := os.Open(partFileName)
+		if err != nil {
+			return err
+		}
+		io.Copy(destFile, partFile)
+		partFile.Close()
+		os.Remove(partFileName)
+	}
+
+	return nil
+}
+
+func (d *Downloader) getPartFilename(filename string, partNum int) string {
+	return fmt.Sprintf("%s/%s-%d", os.TempDir(), filepath.Base(filename), partNum)
+}
+
+func (d *Downloader) singleDownload(strURL, filename string) error {
+	resp, err := http.Get(strURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	d.progress.SetTotal(int64(resp.ContentLength))
+
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	buf := make([]byte, 32*1024)
+	_, err = io.CopyBuffer(io.MultiWriter(f, d.progress), resp.Body, buf)
+	return err
+}

go.mod

@@ -1,10 +1,16 @@
-module github.com/smbrave/goutil
+module git.u8t.cn/open/goutil
 
-go 1.18
+go 1.24
 
 require (
-	github.com/sirupsen/logrus v1.9.0
-	gorm.io/gorm v1.25.5
+	github.com/sirupsen/logrus v1.9.3
+	github.com/speps/go-hashids v2.0.0+incompatible
+	gorm.io/gorm v1.30.2
 )
 
-require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+require (
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/text v0.20.0 // indirect
+)

memory.go

@@ -0,0 +1,74 @@
+package goutil
+
+import (
+	"sync"
+	"time"
+)
+
+// Memory struct contains *memcache.Client
+type Memory struct {
+	sync.Mutex
+
+	data map[string]*data
+}
+
+type data struct {
+	Data    interface{}
+	Expired time.Time
+}
+
+// NewMemory create new memcache
+func NewMemory() *Memory {
+	return &Memory{
+		data: map[string]*data{},
+	}
+}
+
+// Get return cached value
+func (mem *Memory) Get(key string) interface{} {
+	if ret, ok := mem.data[key]; ok {
+		if ret.Expired.Before(time.Now()) {
+			mem.deleteKey(key)
+			return nil
+		}
+		return ret.Data
+	}
+	return nil
+}
+
+// IsExist check value exists in memcache.
+func (mem *Memory) IsExist(key string) bool {
+	if ret, ok := mem.data[key]; ok {
+		if ret.Expired.Before(time.Now()) {
+			mem.deleteKey(key)
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// Set cached value with key and expire time.
+func (mem *Memory) Set(key string, val interface{}, timeout time.Duration) (err error) {
+	mem.Lock()
+	defer mem.Unlock()
+
+	mem.data[key] = &data{
+		Data:    val,
+		Expired: time.Now().Add(timeout),
+	}
+	return nil
+}
+
+// Delete delete value in memcache.
+func (mem *Memory) Delete(key string) error {
+	return mem.deleteKey(key)
+}
+
+// deleteKey
+func (mem *Memory) deleteKey(key string) error {
+	mem.Lock()
+	defer mem.Unlock()
+	delete(mem.data, key)
+	return nil
+}

number.go

@@ -2,6 +2,7 @@ package goutil
 
 import (
 	"crypto/md5"
+	"github.com/speps/go-hashids"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -68,3 +69,19 @@ func Hash32(s string) uint32 {
 	}
 	return digest
 }
+
+func EncryptID(data int64, salt string) string {
+	hd := hashids.NewData()
+	hd.Salt = salt
+	h, _ := hashids.NewWithData(hd)
+	e, _ := h.Encode([]int{int(data)})
+	return e
+}
+
+func DecryptID(data, salt string) int64 {
+	hd := hashids.NewData()
+	hd.Salt = salt
+	h, _ := hashids.NewWithData(hd)
+	e, _ := h.DecodeWithError(data)
+	return int64(e[0])
+}

util.go

@@ -3,10 +3,13 @@ package goutil
 import (
 	"errors"
 	"fmt"
+	"math/rand"
 	"reflect"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // FormatMoney 格式化商品价格
@@ -19,6 +22,13 @@ func FormatMoney(number int64) string {
 	return strconv.FormatInt(int64(num1), 10)
 }
 
+func FormatFloat(f float64) string {
+	if int64(f*100)%100 == 0 {
+		return fmt.Sprintf("%d", int64(f))
+	}
+	return fmt.Sprintf("%.2f", f)
+}
+
 func FormatPercent(number float64) string {
 	if number*100 == float64(int(number*100)) {
 		return fmt.Sprintf("%d%%", int(number*100))
@@ -127,3 +137,48 @@ func HtmlStrip(src string) string {
 	src = strings.ReplaceAll(src, "&nbsp", "")
 	return strings.TrimSpace(src)
 }
+
+func Reverse(s interface{}) {
+	sort.SliceStable(s, func(i, j int) bool {
+		return true
+	})
+}
+
+func WeightKeyword(keywords string) string {
+	keywords = strings.ReplaceAll(keywords, "，", ",")
+	keywords = strings.ReplaceAll(keywords, "：", ":")
+	fields := strings.Split(keywords, ",")
+	if len(fields) == 1 {
+		return keywords
+	}
+
+	sumWeight := int(0)
+	arrWeight := make([]int, 0)
+	arrKey := make([]string, 0)
+
+	for _, field := range fields {
+		field = strings.Trim(field, "\r\t\n ")
+		kvs := strings.SplitN(field, ":", 2)
+		key := strings.Trim(kvs[0], "\r\t\n ")
+		weight := int(0)
+		if len(kvs) > 1 {
+			weight, _ = strconv.Atoi(strings.Trim(kvs[1], "\r\t\n "))
+		}
+		if weight <= 0 {
+			continue
+		}
+		sumWeight += weight
+		arrWeight = append(arrWeight, sumWeight)
+		arrKey = append(arrKey, key)
+	}
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	randWeight := r.Intn(sumWeight)
+
+	for i := 0; i < len(arrWeight); i++ {
+		if randWeight < arrWeight[i] {
+			return arrKey[i]
+		}
+	}
+
+	return ""
+}