sx/TdOrm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Smile-Xin 2024-10-30 15:29:19 +08:00
parent a9efee5504
commit 0d96fe7f54
14 changed files with 69569 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/orm.iml" filepath="$PROJECT_DIR$/.idea/orm.iml" />
</modules>
</component>
</project>

9
.idea/orm.iml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

36
DB/initDB.go Normal file
View File

@ -0,0 +1,36 @@
package DB
import (
"database/sql"
_ "github.com/taosdata/driver-go/v3/taosWS"
"log"
)
var Db *sql.DB
func InitDB() {
// use
// var taosDSN = "root:taosdata@ws(localhost:6041)/dbName"
// if you want to connect a specified database named "dbName"
// 公司局域网服务器
var taosDSN = "root:taosdata@ws(192.168.151.102:6041)/test"
// 远程服务器
//var taosDSN = "root:taosdata@ws(123.249.22.213:6041)/test"
var err error
Db, err = sql.Open("taosWS", taosDSN)
if err != nil {
log.Fatalln("Failed to connect to " + taosDSN + "; ErrMessage: " + err.Error())
}
//fmt.Println("Connected to ", db, " successfully.")
// SetMaxOpenConns sets the maximum number of open connections to the database. 0 means unlimited.
Db.SetMaxOpenConns(0)
// SetMaxIdleConns sets the maximum number of connections in the idle connection pool.
Db.SetMaxIdleConns(2)
// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
Db.SetConnMaxLifetime(5)
// SetConnMaxIdleTime sets the maximum amount of time a connection may be idle.
Db.SetConnMaxIdleTime(10)
}

5
dao/DAO.go Normal file
View File

@ -0,0 +1,5 @@
package dao
func DAO() {
}

1
dao/MetersDAO.go Normal file
View File

@ -0,0 +1 @@
package dao

1
data.csv Normal file
View File

@ -0,0 +1 @@
{"id":1,"local_time":"2024/7/15 0:00:03","pitch_angle":0,"power":2370,"speed":312.1,"ts":"2024-09-14T17:15:51.014+08:00","wind_dir":40.05,"wind_ori":178.141,"wind_speed":8.67004}{"id":2,"local_time":"2024/7/15 0:00:08","pitch_angle":0,"power":2369,"speed":312.6,"ts":"2024-09-14T17:15:51.018+08:00","wind_dir":19.8629,"wind_ori":180.386,"wind_speed":10.1143}{"id":3,"local_time":"2024/7/15 0:00:13","pitch_angle":0,"power":2472,"speed":316.6,"ts":"2024-09-14T17:15:51.021+08:00","wind_dir":21.1923,"wind_ori":179.906,"wind_speed":12.291}{"id":4,"local_time":"2024/7/15 0:00:18","pitch_angle":0,"power":2486,"speed":317.9,"ts":"2024-09-14T17:15:51.025+08:00","wind_dir":12.0731,"wind_ori":178.457,"wind_speed":10.2104}{"id":5,"local_time":"2024/7/15 0:00:23","pitch_angle":0,"power":2410,"speed":314.2,"ts":"2024-09-14T17:15:51.028+08:00","wind_dir":9.37096,"wind_ori":176.586,"wind_speed":9.4368}{"id":6,"local_time":"2024/7/15 0:00:28","pitch_angle":0,"power":2301,"speed":309.6,"ts":"2024-09-14T17:15:51.031+08:00","wind_dir":-0.956185,"wind_ori":174.966,"wind_speed":9.1713}{"id":7,"local_time":"2024/7/15 0:00:33","pitch_angle":0,"power":2127,"speed":309,"ts":"2024-09-14T17:15:51.034+08:00","wind_dir":6.59142,"wind_ori":173.718,"wind_speed":10.3386}{"id":8,"local_time":"2024/7/15 0:00:38","pitch_angle":0,"power":2105,"speed":307.4,"ts":"2024-09-14T17:15:51.037+08:00","wind_dir":-3.49403,"wind_ori":172.815,"wind_speed":9.83047}{"id":9,"local_time":"2024/7/15 0:00:48","pitch_angle":0,"power":2096,"speed":307.3,"ts":"2024-09-14T17:15:51.046+08:00","wind_dir":-11.7545,"wind_ori":169.392,"wind_speed":8.1665}{"id":10,"local_time":"2024/7/15 0:00:53","pitch_angle":0,"power":2102,"speed":307.4,"ts":"2024-09-14T17:15:51.049+08:00","wind_dir":-3.62459,"wind_ori":167.385,"wind_speed":8.86002}
Can't render this file because it contains an unexpected character in line 1 and column 2.

13
go.mod Normal file
View File

@ -0,0 +1,13 @@
module orm
go 1.23.0
require github.com/taosdata/driver-go/v3 v3.5.8
require (
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
)

25
go.sum Normal file
View File

@ -0,0 +1,25 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/taosdata/driver-go/v3 v3.5.8 h1:JT5lNFUCOHD9Hs4Phjg8RBkGOWlePRnpGqq8kIRHT98=
github.com/taosdata/driver-go/v3 v3.5.8/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

49
main.go Normal file
View File

@ -0,0 +1,49 @@
package main
import (
"orm/DB"
"orm/serve"
)
func main() {
DB.InitDB()
//now := time.Now()
//for i := 0; i < 50; i++ {
// serve.Serve()
//}
//since := time.Since(now)
//fmt.Println("time", since/50)
serve.Serve()
DB.Db.Close()
//dial, err := net.Dial("tcp", "192.168.151.135:58565")
//if err != nil {
// panic(err)
//}
//fmt.Println("remoteAddr", dial.RemoteAddr().String())
//fmt.Println("localAddr", dial.LocalAddr().String())
//listenAddr := "127.0.0.1:58565"
//listen, err := net.Listen("tcp", listenAddr)
//if err != nil {
// panic(err)
//}
//defer listen.Close()
//accept, err := listen.Accept()
//if err != nil {
// panic(err)
//}
//
//// 读数据
//reader := bufio.NewReader(accept)
//readString, err := reader.ReadString('\n')
//if err != nil {
// panic(err)
//}
//fmt.Println("readString", readString)
//accept.Close()
}

26
model/Meters.go Normal file
View File

@ -0,0 +1,26 @@
package model
import "time"
//type Meters struct {
// Ts time.Time
// Current float64
// Voltage int
// Phase float64
// Groupid int
// Location string
//}
type Meters struct {
Ts time.Time
Local_time string
Wind_speed float64
Id int
Pitch_angle float64
Wind_dir float64
Wind_ori float64
Power float64
Speed float64
Groupid int
Ip_addr string
}

341
serve/Serve.go Normal file
View File

@ -0,0 +1,341 @@
package serve
import (
"encoding/json"
"fmt"
"io"
"orm/DB"
"os"
"strings"
"sync"
"time"
)
func getParams(fieldName []string, queryParam map[string]string) {
// 获取参数
}
func Serve() {
table := "tongbai_wt16"
selectToCsvFile(table)
// 创建数据库
//createTable()
// 通过csv文件插入数据
//allStart := time.Now()
//for i := 0; i < 100; i++ {
//insertByCsv("./test.csv")
//}
//allSince := time.Since(allStart)
//fmt.Println("总时间", allSince)
}
// 读取数据库数据并写入文件
func selectToCsvFile(table string) {
// 获取sql语句
sql := fmt.Sprintf("select * from `lxc_db`.`%s`", table)
//sql := "select * from `lxc_db`.`tongbai_wt11` limit 100"
//sql = "select * from `test`.`meters` limit 10000"
// 执行select语句
mData := selectExec(sql)
// 写入文件
writeToFile(mData, "./test.csv")
}
// 执行select语句
func selectExec(sql string) []map[string]interface{} {
//now := time.Now()
// 执行sql语句
query, err := DB.Db.Query(sql)
if err != nil {
panic(err)
}
//since := time.Since(now)
// 数据列
columns, err := query.Columns()
if err != nil {
panic(err)
}
// 数据个数
count := len(columns)
// 返回值 Map切片
mData := make([]map[string]interface{}, 0)
// 一条数据的各列的值（需要指定长度为列的个数，以便获取地址）
values := make([]interface{}, count)
// 一条数据的各列的值的地址
valPointers := make([]interface{}, count)
// 处理查询结果
for query.Next() {
// 获取各列的值的地址
for i := 0; i < count; i++ {
valPointers[i] = &values[i]
}
// 获取各列的值，放到对应的地址中
query.Scan(valPointers...)
// 一条数据的Map (列名和值的键值对)
entry := make(map[string]interface{})
// Map 赋值
for i, col := range columns {
var v interface{}
// 值复制给val(所以Scan时指定的地址可重复使用)
val := values[i]
b, ok := val.([]byte)
if ok {
// 字符切片转为字符串
v = string(b)
} else {
v = val
}
entry[col] = v
}
mData = append(mData, entry)
}
return mData
// 循环打印mData
//for _, v := range mData {
// fmt.Println(v)
//}
}
// 判断文件是否存在
func fileExist(path string) bool {
// 判断文件是否存在
_, err := os.Stat(path)
if err != nil {
return false
}
return true
}
// 将数据写入文件
func writeToFile(mData []map[string]interface{}, path string) {
// 写入文件的字符串
var fileStr string
// 创建表头
var fileHead []string
// 创建文件
var csvFile *os.File
var err error
defer csvFile.Close()
// 判断文件是否存在
if fileExist(path) {
file := readCsvFile(path)
if file == "" || file == "\n" {
} else {
csvFile, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
panic(err)
}
fileLine := strings.Split(file, "\n")
fileHead = strings.Split(strings.ReplaceAll(fileLine[0], "\"", ""), ",")
fmt.Println("fileHead", fileHead)
}
} else {
csvFile, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
panic(err)
}
for k := range mData[0] {
// 将key转为字符串
key, err := json.Marshal(k)
if err != nil {
panic(err)
}
// 将key转为字符串后，拼接到fileStr
fileStr += string(key)
fileStr += ","
// 向fileHead添加key
fileHead = append(fileHead, k)
fmt.Println("fileHead", fileHead)
}
// 去掉最后一个逗号，此时fileStr第一行为表头
fileStr = strings.TrimRight(fileStr, ",")
fileStr += "\n"
}
// 根据表头读数据并写入文件
// 将mData写入文件 循环[]map[string]interface{}
dataPoint := 0
go func() {
// 写入文件
if _, err := io.WriteString(csvFile, fileStr); err != nil {
fmt.Println("写入文件失败： ", err)
os.Exit(1)
}
}()
for {
if len(mData)-dataPoint < 500 {
fmt.Println("数据量小于500")
fileStr += getStrHandle(mData[dataPoint:], fileHead)
break
}
go func() {
fileStr += getStrHandle(mData[dataPoint:dataPoint+500], fileHead)
dataPoint += 500
}()
}
// 循环表头，根据表头读每一行的map的数据
// 删除最后一个换行符
//fileStr = strings.TrimRight(fileStr, "\n")
//fmt.Println(fileStr)
//getSqlByContent(fileStr)
// 写入文件
if _, err := io.WriteString(csvFile, fileStr); err != nil {
fmt.Println("写入文件失败： ", err)
os.Exit(1)
}
}
func getStrHandle(mData []map[string]interface{}, fileHead []string) (fileStr string) {
now := time.Now()
for _, mapEntry := range mData {
// 循环表头，根据表头读每一行的map的数据
for _, key := range fileHead {
if key == "ts" {
now = now.Add(time.Hour * 1)
fileStr += "'" + now.Format("2006-01-02 15:04:05") + "'"
//value, err := json.Marshal(mapEntry[key])
//if err != nil {
// panic(err)
//}
//fileStr += string(value)
} else {
value, err := json.Marshal(mapEntry[key])
if err != nil {
panic(err)
}
fileStr += string(value)
}
fileStr += ","
}
fileStr = strings.TrimRight(fileStr, ",")
fileStr += "\n"
fmt.Println("写完一行数据")
}
// 删除最后一个换行符
//fileStr = strings.TrimRight(fileStr, "\n")
//fmt.Println(fileStr)
//getSqlByContent(fileStr)
return
}
// 通过csv文件插入数据
func insertByCsv(filePath string) {
// 将文件写入数据库, 试用版不支持
//sql := fmt.Sprintf("INSERT INTO `lxc_db`.`tongbai_wt12` FILE %s;", filePath)
//_, err := DB.Db.Exec(sql)
//if err != nil {
// panic(err)
//}
// 读取文件冰生成sql语句
// 读取文件
readFront := time.Now()
fileContent := readCsvFile(filePath)
readSince := time.Since(readFront)
fmt.Println("读文件所用时间", readSince)
// 生成写入数据库的sql语句
sqlFront := time.Now()
sqlList := getSqlByContent(fileContent)
sqlSince := time.Since(sqlFront)
fmt.Println("生成sql语句所用时间", sqlSince)
// 执行sql语句
//fmt.Println("sql", sql)
execFront := time.Now()
wg := sync.WaitGroup{}
for _, sql := range sqlList {
wg.Add(1)
go func() {
_, err := DB.Db.Exec(sql)
if err != nil {
}
wg.Done()
}()
//_, err := DB.Db.Exec(sql)
//if err != nil {
// panic(err)
//} else {
// //fmt.Println("插入成功")
//}
}
wg.Wait()
execSince := time.Since(execFront)
fmt.Println("执行sql语句所用时间", execSince)
}
// 读取csv文件
func readCsvFile(filePath string) string {
// 打开文件
csvFile, err := os.OpenFile(filePath, os.O_RDONLY, 0666)
if err != nil {
panic(err)
}
defer csvFile.Close()
// 读取CSV文件
fileContent, err := io.ReadAll(csvFile)
if err != nil {
panic(err)
}
return string(fileContent)
}
// 根据文件内容生成sql语句
func getSqlByContent(fileContent string) []string {
// 将文件内容按行分割
dataLine := strings.Split(fileContent, "\n")
// 第一行为表头
head := dataLine[0]
// 剩余行为数据
content := dataLine[1:]
var data string
var sqlList []string
var l int
for i, v := range content {
content[i] = "(" + v + ")"
data += content[i]
l++
// 每500条数据生成一条sql语句
if l > 500 {
l = 0
sql := fmt.Sprintf("INSERT INTO `lxc_db`.`tongbai_wt16` (%s) VALUES %s;", head, data)
sqlList = append(sqlList, sql)
data = ""
}
}
// 生成sql语句
sql := fmt.Sprintf("INSERT INTO `lxc_db`.`tongbai_wt16` (%s) VALUES %s;", head, data)
sqlList = append(sqlList, sql)
return sqlList
}
// 创建表
func createTable() {
// 创建表
sql := "CREATE TABLE `lxc_db`.`tongbai_wt16` USING `lxc_db`.`meters` (`groupid`, `ip_addr`) TAGS (10011, NULL)"
exec, err := DB.Db.Exec(sql)
if err != nil {
panic(err)
}
id, err := exec.RowsAffected()
if err != nil {
panic(err)
}
fmt.Println("exec.RowsAffected", id)
}

69041
test.csv Normal file
View File

File diff suppressed because it is too large Load Diff