open/servicebase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

first commit

Browse Source
This commit is contained in:
Yangtao
2025-11-18 17:48:20 +08:00
commit 6e56cab848
196 changed files with 65809 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
pkg/common/es/document/user.go Normal file
View File

@ -0,0 +1,94 @@
package document
type User struct {
Id string
Mobile string
RegionCode string
NickName string
Salt string
Password string
Status string
Avatar string
Gender string
Birthday string
WxUnionId string
QqOpenId string
MarketChannel string
CreateTime string
IsActive string
ActiveTime string
SignupSource string
SignupDevicePlatform string
SignupDeviceUdid string
IdNo string
IsAuth string
TrueName string
Sign string
ViewFlag string
VideoUrl string
School string
UserNo string
BankName string
BankUserName string
BankBranchName string
BankCardNo string
AuthFailReason string
AlipayAccount string
WellNoIcon string
}
//字符串: text keyword
//整数 : byte, short, integer, long
//浮点数: float, double
//布尔型: boolean
//日期: date
var UserIndex = map[string]interface{}{
"settings": map[string]interface{}{"number_of_shards": 5, "number_of_replicas": 1},
"mappings": map[string]interface{}{
"properties": map[string]interface{}{
"ActiveTime": map[string]string{"type": "date", "format": "yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd HH:mm:ss.SSSZ||yyyy-MM-ddZ||epoch_millis||epoch_second"},
"AlipayAccount": map[string]string{"type": "keyword"},
"AuthFailReason": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"Avatar": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"BankBranchName": map[string]string{"type": "keyword"},
"BankCardNo": map[string]string{"type": "keyword"},
"BankName": map[string]string{"type": "text"},
"BankUserName": map[string]string{"type": "text"},
"Birthday": map[string]string{"type": "date", "format": "yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd HH:mm:ss.SSSZ||yyyy-MM-ddZ||epoch_millis||epoch_second"},
"CreateTime": map[string]string{"type": "date", "format": "yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd HH:mm:ss.SSSZ||yyyy-MM-ddZ||epoch_millis||epoch_second"},
"Gender": map[string]string{"type": "keyword"},
"Id": map[string]string{"type": "keyword"},
"IdNo": map[string]string{"type": "keyword"},
"IsActive": map[string]string{"type": "keyword"},
"IsAuth": map[string]string{"type": "keyword"},
"MarketChannel": map[string]string{"type": "keyword"},
"Mobile": map[string]string{"type": "keyword"},
"NickName": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"QqOpenId": map[string]string{"type": "keyword"},
"RegionCode": map[string]string{"type": "keyword"},
"Salt": map[string]string{"type": "keyword"},
"School": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"Sign": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"SignupDevicePlatform": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"SignupDeviceUdid": map[string]string{"type": "keyword"},
"SignupSource": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"Status": map[string]string{"type": "keyword"},
"TrueName": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"UserNo": map[string]string{"type": "keyword"},
"VideoUrl": map[string]string{"type": "text", "analyzer": "ik_max_word"},
"ViewFlag": map[string]string{"type": "keyword"},
"WellNoIcon": map[string]string{"type": "keyword"},
"WxUnionId": map[string]string{"type": "keyword"},
},
},
}
var MessageIndex = map[string]interface{}{
"settings": map[string]interface{}{"number_of_shards": 5, "number_of_replicas": 1},
"mappings": map[string]interface{}{
"properties": map[string]interface{}{
"id": map[string]string{"type": "keyword"},
"curTime": map[string]string{"type": "date", "format": "yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd HH:mm:ss.SSSZ||yyyy-MM-ddZ||epoch_millis||epoch_second"},
},
},
}

261
pkg/common/es/es_client.go Normal file
View File

@ -0,0 +1,261 @@
package ess
import (
"servicebase/pkg/common/es/document"
"servicebase/pkg/common/messages"
"context"
"encoding/json"
"reflect"
"time"
"github.com/anxpp/beego/logs"
"github.com/olivere/elastic/v7"
"github.com/spf13/viper"
)
var client *elastic.Client
type EsClient struct {
}
func Init() {
index := messages.TagIndex(string(messages.EventTagUser))
client = _connect()
exists, e := client.IndexExists(index).Do(context.Background())
if e != nil {
logs.Error("IndexExists ", index, " error: ", e.Error())
return
}
if !exists {
logs.Info("es user not exist")
createIndex, e := client.CreateIndex(index).BodyJson(document.UserIndex).Do(context.Background())
if e != nil {
logs.Error("IndexCreate ", index, " error: ", e.Error())
return
}
if !createIndex.Acknowledged {
}
}
indexMessage := messages.TagIndex(string(messages.EventTagMessage))
existsMessage, e := client.IndexExists(indexMessage).Do(context.Background())
if e != nil {
logs.Error("IndexExists indexMessage", index, " error: ", e.Error())
return
}
if !existsMessage {
logs.Info("es indexMessage not exist")
createIndex, e := client.CreateIndex(indexMessage).BodyJson(document.MessageIndex).Do(context.Background())
if e != nil {
logs.Error("IndexCreate ", indexMessage, " error: ", e.Error())
return
}
if !createIndex.Acknowledged {
}
}
logs.Info("init elasticsearch finish")
}
func _connect() (c *elastic.Client) {
c, e := elastic.NewSimpleClient(
elastic.SetHealthcheck(true),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetURL(viper.GetString("es.default.addr")),
// elastic.SetBasicAuth(beego.AppConfig.String("es_username"), beego.AppConfig.String("es_password")),
)
if e != nil {
logs.Error("NewClient_error: ", e.Error())
}
return c
}
// 创建记录
func (*EsClient) Create(index, id string, model interface{}) (body string, success bool, msg string) {
client := _connect()
_, e := client.Index().Index(index).Id(id).BodyJson(model).Do(context.Background())
if e != nil {
success = false
msg = e.Error()
b, _ := json.Marshal(model)
logs.Error("Document Create Error: ", e.Error(), " document: ", string(b), index, id)
return
}
success = true
return
}
// 判断存在
func (*EsClient) Exists(index, id string) (exists, success bool, msg string, e error) {
client := _connect()
exists, e = client.Exists().Index(index).Type("_doc").Id(id).Do(context.Background())
if e != nil {
success = false
msg = e.Error()
logs.Error("Id Exists Error: ", e.Error(), " id: ", id)
return
}
success = true
return
}
// 更新记录
func (*EsClient) Update(index, id string, model interface{}) (body string, success bool, msg string) {
client := _connect()
_, e := client.Update().Index(index).Id(id).Doc(model).Do(context.Background())
if e != nil {
success = false
msg = e.Error()
return
}
success = true
return
}
// 搜索
func (*EsClient) Search(index string, key string, fields ...string) (body []interface{}, success bool, msg string) {
client := _connect()
logs.Info("ES Search index =", index, " key =", key, " fields =", fields)
var list []*elastic.WildcardQuery
for _, field := range fields {
list = append(list, elastic.NewWildcardQuery(field, key))
}
s := client.Search().Index(index)
for _, query := range list {
s = s.Query(query)
}
res, e := s.From(0).Size(20).Pretty(true).Do(context.Background())
if e != nil {
success = false
msg = e.Error()
return
}
logs.Info(res.Status)
logs.Info(res.Hits.TotalHits)
var item document.User
for _, item := range res.Each(reflect.TypeOf(item)) {
if t, ok := item.(document.User); ok {
body = append(body, t)
}
}
//if res.Hits.TotalHits.Value > 0 {
// for _, hit := range res.Hits.Hits {
// var t document.User
// _ := json.Unmarshal(hit.Source, &t)
// body = append(body, t)
// }
//}
return
}
type ESFilter struct {
Queries []ESQuery
BoolMustInShouldQueries [][]ESQuery
Sort ESSort
}
type ESQuery struct {
Key string
Value string
Type string
}
type ESSort struct {
Field string
Ascending bool
}
// 搜索消息
func (*EsClient) SearchMulti(index string, filter ESFilter, page, size int) (result interface{}, success bool, msg string) {
client := _connect()
var list []elastic.Query
for _, item := range filter.Queries {
switch item.Type {
case "match":
list = append(list, elastic.NewMatchQuery(item.Key, item.Value))
case "multi_match":
list = append(list, elastic.NewMultiMatchQuery(item.Value).Type("best_fields").Lenient(true))
case "range_gte":
list = append(list, elastic.NewRangeQuery(item.Key).Gte(item.Value))
case "range_lte":
list = append(list, elastic.NewRangeQuery(item.Key).Lte(item.Value))
}
}
if len(filter.BoolMustInShouldQueries) > 0 {
BQ := elastic.NewBoolQuery()
var bqList []elastic.Query
for _, bq := range filter.BoolMustInShouldQueries {
var listBq []elastic.Query
boolQuery := elastic.NewBoolQuery()
for _, item := range bq {
switch item.Type {
case "match":
listBq = append(listBq, elastic.NewMatchQuery(item.Key, item.Value))
case "multi_match":
listBq = append(listBq, elastic.NewMultiMatchQuery(item.Value).Type("best_fields").Lenient(true))
case "range_gte":
listBq = append(listBq, elastic.NewRangeQuery(item.Key).Gte(item.Value))
case "range_lte":
listBq = append(listBq, elastic.NewRangeQuery(item.Key).Lte(item.Value))
}
}
boolQuery.Must(listBq...)
bqList = append(bqList, boolQuery)
}
BQ.Should(bqList...)
list = append(list, BQ)
}
query := elastic.NewBoolQuery().Filter(list...)
//logs.Info(len(list))
//a1, _ := query.Source()
//a2, _ := json.MarshalIndent(a1, "", " ")
//logs.Info(string(a2))
s := client.Search().Index(index).Query(query)
if len(filter.Sort.Field) > 0 {
s = s.Sort(filter.Sort.Field, filter.Sort.Ascending)
}
res, e := s.From(page * size).Size(size).Pretty(false).Do(context.Background())
if e != nil {
return
}
var body []interface{}
for _, item := range res.Hits.Hits {
b, _ := item.Source.MarshalJSON()
m := make(map[string]interface{})
_ = json.Unmarshal(b, &m)
body = append(body, m)
}
result = map[string]interface{}{
"Status": res.Status,
"Total": res.Hits.TotalHits,
"List": body,
}
return
}
// 搜索
func (*EsClient) QueryString(page, size int, index string, key string, fields ...string) (list []document.User, count int64, success bool, msg string) {
client := _connect()
logs.Info("ES QueryString index =", index, " key =", key, " fields =", fields)
query := elastic.NewQueryStringQuery(key)
for _, field := range fields {
query.Field(field)
}
query.AnalyzeWildcard(false)
s := client.Search().Index(index)
s = s.Query(query)
s = s.Sort("_score", false)
res, e := s.From(page * size).Size(size).Pretty(true).Do(context.Background())
if e != nil {
success = false
msg = e.Error()
return
}
count = res.TotalHits()
var user document.User
for _, item := range res.Each(reflect.TypeOf(user)) {
if t, ok := item.(document.User); ok {
list = append(list, t)
}
}
success = true
return
}

81
pkg/common/es/es_client_test.go Normal file
View File

@ -0,0 +1,81 @@
package ess
import (
"servicebase/pkg/common/messages"
"context"
"testing"
"time"
"github.com/anxpp/beego/logs"
"github.com/olivere/elastic/v7"
"github.com/spf13/viper"
)
func TestClient(t *testing.T) {
Init()
}
func TestClientCreateUser(t *testing.T) {
//for _, item := range models.LogUserRegisterAll() {
// cli := EsClient{}
// _, success, msg := cli.Create("test_user_register", item.Id, item)
// if !success {
// logs.Error(msg)
// }
//}
}
func TestClientSearch(t *testing.T) {
}
func _testConnect() (c *elastic.Client) {
c, e := elastic.NewSimpleClient(
elastic.SetHealthcheck(true),
elastic.SetHealthcheckInterval(10*time.Second),
// elastic.SetURL("http://47.97.157.234:9200"),
elastic.SetURL(viper.GetString("es.default.addr")),
// elastic.SetBasicAuth(beego.AppConfig.String("es_username"), beego.AppConfig.String("es_password")),
)
if e != nil {
logs.Error("NewClient_error: ", e.Error())
}
return c
}
func TestClientNew(t *testing.T) {
var client EsClient
client.Create("es_index_message", "003", map[string]interface{}{
"id": "003",
"curTime": 1440570500855,
"f1": 1234,
"f2": "22222",
"f3": "22222",
"f4": "22222",
"f5": "22222",
"f6": "2020-08-09 23:37:00",
})
}
func TestSearchMessage(t *testing.T) {
index := messages.TagIndex(string(messages.EventTagMessage))
client := _testConnect()
var list []elastic.Query
list = append(list, elastic.NewWildcardQuery("eventType", "1"))
list = append(list, elastic.NewMatchQuery("msgType", "TEXT"))
s := client.Search().Index(index)
for _, query := range list {
s = s.Query(query)
}
res, e := s.From(0).Size(20).Pretty(true).Do(context.Background())
if e != nil {
return
}
logs.Info(res.Status)
logs.Info(res.Hits.TotalHits)
logs.Info(len(res.Hits.Hits))
for i, item := range res.Hits.Hits {
b, _ := item.Source.MarshalJSON()
logs.Info(i, string(b))
}
time.Sleep(time.Second * 3)
}