优化相似度流程

log.ini

@@ -1,5 +1,5 @@
-dir=logs
+dir=log
 match=warn
-url=http://192.168.102.21:6801/upload_v2
+url=http://192.168.22.74:6801/upload_v2
 key=测试
 info=测试

main.go

@@ -21,7 +21,7 @@ func main() {
 		fmt.Println(err)
 		return
 	}
-
+	var okList []string
 	for _, item := range dirs {
 		if regex.MatchString(item.Name()) {
 			file, err := os.Open(filepath.Join(dir, item.Name()))
@@ -34,15 +34,23 @@ func main() {
 			})
 			file.Close()
 			for _, text := range logTexts {
+				if calculateSimilaritys(1, text, okList...) {
+					continue
+				}
 				err := requestDingApi(url, message{
 					Text:     text,
 					Filename: item.Name(),
 					Key:      key,
 					Info:     info,
 				})
-				fmt.Println(err)
+				if err != nil {
+					fmt.Println(err)
+					fmt.Println("日志服务器关闭")
+					return
+				}
+				okList = append(okList, text)
 			}
 		}
 	}
-
+	fmt.Println("完成上报数量：", len(okList))
 }

pauseErlang.go

@@ -0,0 +1,76 @@
+package main
+
+// 是否有符合要求的log
+func calculateSimilaritys(n int, log1 string, logs ...string) bool {
+	for _, log := range logs {
+		similarity := calculateSimilarity(log1, log)
+		if similarity <= n {
+			return true
+		}
+	}
+	return false
+}
+
+// 放回不相同数量
+func calculateSimilarity(log1, log2 string) int {
+	l1 := parseErlangString(log1)
+	l2 := parseErlangString(log2)
+	s, l := calcSimilar(l1, l2)
+	return l - s
+}
+
+// 计算相似度
+func calcSimilar(log1 []string, log2 []string) (int, int) {
+	//忽略
+	const ignore = 4
+	langer := max(len(log1), len(log2))
+	later := min(len(log1), len(log2))
+	sim := 0
+	for i := ignore; i < later; i++ {
+		if log1[i] == log2[i] {
+			sim++
+		} else if len(log1[i]) == len(log2[i]) {
+			var iSim = 0
+			for j := 0; j < len(log1[i]); j++ {
+				if log1[i][j] == log2[i][j] {
+					iSim++
+				}
+			}
+			if float64(iSim)/float64(len(log1[i])) > 0.95 {
+				sim++
+			}
+		}
+	}
+	return sim, langer - 4
+}
+
+func parseErlangString(logStr string) []string {
+	var (
+		head = 0
+		i    []int32
+		r    []string
+	)
+	for _, v := range logStr {
+		switch v {
+		case ' ':
+		case '{':
+			head++
+			if head != 1 {
+				i = append(i, v)
+			}
+		case '}':
+			if head != 1 {
+				i = append(i, v)
+			}
+			head--
+		default:
+			if v == ',' && len(i) != 0 && head == 1 {
+				r = append(r, string(i))
+				i = nil
+			} else {
+				i = append(i, v)
+			}
+		}
+	}
+	return append(r, string(i))
+}