mirror of
https://github.com/ultravioletrs/cocos.git
synced 2026-06-23 04:10:25 +00:00
Merge remote-tracking branch 'agent/main'
Signed-off-by: Drasko Draskovic <drasko.draskovic@gmail.com>
This commit is contained in:
Drasko Draskovic
@@ -0,0 +1,67 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "agent/agent.proto"
|
||||
- "agent/*.pb.go"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "agent/agent.proto"
|
||||
- "agent/*.pb.go"
|
||||
|
||||
jobs:
|
||||
checkproto:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.20.x
|
||||
|
||||
- name: Set up protoc
|
||||
run: |
|
||||
PROTOC_VERSION=23.3
|
||||
PROTOC_GEN_VERSION=v1.31.0
|
||||
PROTOC_GRPC_VERSION=v1.3.0
|
||||
|
||||
# Download and install protoc
|
||||
PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip
|
||||
curl -0L -o $PROTOC_ZIP https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP
|
||||
unzip -o $PROTOC_ZIP -d protoc3
|
||||
sudo mv protoc3/bin/* /usr/local/bin/
|
||||
sudo mv protoc3/include/* /usr/local/include/
|
||||
rm -rf $PROTOC_ZIP protoc3
|
||||
|
||||
# Install protoc-gen-go and protoc-gen-go-grpc
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@$PROTOC_GEN_VERSION
|
||||
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$PROTOC_GRPC_VERSION
|
||||
|
||||
# Add protoc to the PATH
|
||||
export PATH=$PATH:/usr/local/bin/protoc
|
||||
|
||||
- name: Set up Mainflux
|
||||
run: |
|
||||
# Rename .pb.go files to .pb.go.tmp to prevent conflicts
|
||||
for p in $(ls agent/*.pb.go); do
|
||||
mv $p $p.tmp
|
||||
done
|
||||
|
||||
# Generate Go files from protobuf definitions
|
||||
make protoc
|
||||
|
||||
# Compare generated Go files with the original ones
|
||||
for p in $(ls agent/*.pb.go); do
|
||||
if ! cmp -s $p $p.tmp; then
|
||||
echo "Proto file and generated Go file $p are out of sync!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,36 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.20.x
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
version: latest
|
||||
args: --no-config --disable-all --enable gosimple --enable errcheck --enable govet --enable unused --enable deadcode --timeout 3m
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
go build -mod=vendor -ldflags="-linkmode=external -extldflags=-static -s -w" cmd/agent/main.go
|
||||
go build -mod=vendor -ldflags "-s -w" cmd/cli/main.go
|
||||
|
||||
- name: Run tests
|
||||
run: go test -mod=vendor -v --race -covermode=atomic -coverprofile cover.out ./...
|
||||
@@ -0,0 +1 @@
|
||||
build
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
https://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2015-2020 Mainflux
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,30 @@
|
||||
BUILD_DIR = build
|
||||
SERVICES = agent cli
|
||||
CGO_ENABLED ?= 0
|
||||
GOARCH ?= amd64
|
||||
VERSION ?= $(shell git describe --abbrev=0 --tags)
|
||||
COMMIT ?= $(shell git rev-parse HEAD)
|
||||
TIME ?= $(shell date +%F_%T)
|
||||
CLI_SOURCE = ./cmd/cli/main.go
|
||||
CLI_BIN = ${BUILD_DIR}/cocos-cli
|
||||
|
||||
|
||||
define compile_service
|
||||
CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
||||
go build -mod=vendor -ldflags "-s -w \
|
||||
-X 'github.com/mainflux/mainflux.BuildTime=$(TIME)' \
|
||||
-X 'github.com/mainflux/mainflux.Version=$(VERSION)' \
|
||||
-X 'github.com/mainflux/mainflux.Commit=$(COMMIT)'" \
|
||||
-o ${BUILD_DIR}/cocos-$(1) cmd/$(1)/main.go
|
||||
endef
|
||||
|
||||
.PHONY: all $(SERVICES)
|
||||
|
||||
$(SERVICES):
|
||||
$(call compile_service,$(@))
|
||||
|
||||
install-cli: cli
|
||||
cp ${CLI_BIN} ~/.local/bin/cocos-cli
|
||||
|
||||
protoc:
|
||||
protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative agent/agent.proto
|
||||
@@ -0,0 +1,41 @@
|
||||
# Agent
|
||||
|
||||
Agent service provides a barebones HTTP and gRPC API and Service interface implementation for the development of the agent service.
|
||||
|
||||
## Configuration
|
||||
|
||||
The service is configured using the environment variables from the following table. Note that any unset variables will be replaced with their default values.
|
||||
|
||||
| Variable | Description | Default |
|
||||
| ---------------------- | ------------------------------------------------------ | ------------------------------ |
|
||||
| AGENT_LOG_LEVEL | Log level for agent service (debug, info, warn, error) | info |
|
||||
| AGENT_HTTP_HOST | Agent service HTTP host | "" |
|
||||
| AGENT_HTTP_PORT | Agent service HTTP port | 9031 |
|
||||
| AGENT_HTTP_SERVER_CERT | Path to HTTP server certificate in pem format | "" |
|
||||
| AGENT_HTTP_SERVER_KEY | Path to HTTP server key in pem format | "" |
|
||||
| AGENT_GRPC_HOST | Agent service gRPC host | "" |
|
||||
| AGENT_GRPC_PORT | Agent service gRPC port | 7002 |
|
||||
| AGENT_GRPC_SERVER_CERT | Path to gRPC server certificate in pem format | "" |
|
||||
| AGENT_GRPC_SERVER_KEY | Path to gRPC server key in pem format | "" |
|
||||
| AGENT_JAEGER_URL | Jaeger server URL | http://jaeger:14268/api/traces |
|
||||
|
||||
## Deployment
|
||||
|
||||
To start the service outside of the container, execute the following shell script:
|
||||
|
||||
```bash
|
||||
# download the latest version of the service
|
||||
go get github.com/ultravioletrs/agent
|
||||
|
||||
cd $GOPATH/src/github.com/ultravioletrs/agent
|
||||
|
||||
# compile the agent
|
||||
make agent
|
||||
|
||||
# set the environment variables and run the service
|
||||
./build/cocos-agent
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
For more information about service capabilities and its usage, please check out the [README documentation](../README.md).
|
||||
@@ -0,0 +1,598 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.12.4
|
||||
// source: agent/agent.proto
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
proto "github.com/golang/protobuf/proto"
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// Verify that this generated code is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
|
||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
// This is a compile-time assertion that a sufficiently up-to-date version
|
||||
// of the legacy proto package is being used.
|
||||
const _ = proto.ProtoPackageIsVersion4
|
||||
|
||||
type RunRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Computation []byte `protobuf:"bytes,1,opt,name=computation,proto3" json:"computation,omitempty"`
|
||||
}
|
||||
|
||||
func (x *RunRequest) Reset() {
|
||||
*x = RunRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *RunRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*RunRequest) ProtoMessage() {}
|
||||
|
||||
func (x *RunRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[0]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use RunRequest.ProtoReflect.Descriptor instead.
|
||||
func (*RunRequest) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
func (x *RunRequest) GetComputation() []byte {
|
||||
if x != nil {
|
||||
return x.Computation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RunResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Computation string `protobuf:"bytes,1,opt,name=Computation,proto3" json:"Computation,omitempty"`
|
||||
}
|
||||
|
||||
func (x *RunResponse) Reset() {
|
||||
*x = RunResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *RunResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*RunResponse) ProtoMessage() {}
|
||||
|
||||
func (x *RunResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use RunResponse.ProtoReflect.Descriptor instead.
|
||||
func (*RunResponse) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *RunResponse) GetComputation() string {
|
||||
if x != nil {
|
||||
return x.Computation
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type AlgoRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Algorithm []byte `protobuf:"bytes,1,opt,name=algorithm,proto3" json:"algorithm,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AlgoRequest) Reset() {
|
||||
*x = AlgoRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *AlgoRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*AlgoRequest) ProtoMessage() {}
|
||||
|
||||
func (x *AlgoRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use AlgoRequest.ProtoReflect.Descriptor instead.
|
||||
func (*AlgoRequest) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *AlgoRequest) GetAlgorithm() []byte {
|
||||
if x != nil {
|
||||
return x.Algorithm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AlgoResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
AlgorithmID string `protobuf:"bytes,1,opt,name=algorithmID,proto3" json:"algorithmID,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AlgoResponse) Reset() {
|
||||
*x = AlgoResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[3]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *AlgoResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*AlgoResponse) ProtoMessage() {}
|
||||
|
||||
func (x *AlgoResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[3]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use AlgoResponse.ProtoReflect.Descriptor instead.
|
||||
func (*AlgoResponse) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{3}
|
||||
}
|
||||
|
||||
func (x *AlgoResponse) GetAlgorithmID() string {
|
||||
if x != nil {
|
||||
return x.AlgorithmID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type DataRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"`
|
||||
}
|
||||
|
||||
func (x *DataRequest) Reset() {
|
||||
*x = DataRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[4]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *DataRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*DataRequest) ProtoMessage() {}
|
||||
|
||||
func (x *DataRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[4]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use DataRequest.ProtoReflect.Descriptor instead.
|
||||
func (*DataRequest) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{4}
|
||||
}
|
||||
|
||||
func (x *DataRequest) GetDataset() []byte {
|
||||
if x != nil {
|
||||
return x.Dataset
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type DataResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
DatasetID string `protobuf:"bytes,1,opt,name=datasetID,proto3" json:"datasetID,omitempty"`
|
||||
}
|
||||
|
||||
func (x *DataResponse) Reset() {
|
||||
*x = DataResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[5]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *DataResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*DataResponse) ProtoMessage() {}
|
||||
|
||||
func (x *DataResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[5]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use DataResponse.ProtoReflect.Descriptor instead.
|
||||
func (*DataResponse) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{5}
|
||||
}
|
||||
|
||||
func (x *DataResponse) GetDatasetID() string {
|
||||
if x != nil {
|
||||
return x.DatasetID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type ResultRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
}
|
||||
|
||||
func (x *ResultRequest) Reset() {
|
||||
*x = ResultRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[6]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *ResultRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ResultRequest) ProtoMessage() {}
|
||||
|
||||
func (x *ResultRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[6]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ResultRequest.ProtoReflect.Descriptor instead.
|
||||
func (*ResultRequest) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{6}
|
||||
}
|
||||
|
||||
type ResultResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
File []byte `protobuf:"bytes,1,opt,name=file,proto3" json:"file,omitempty"`
|
||||
}
|
||||
|
||||
func (x *ResultResponse) Reset() {
|
||||
*x = ResultResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_agent_proto_msgTypes[7]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *ResultResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ResultResponse) ProtoMessage() {}
|
||||
|
||||
func (x *ResultResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_agent_proto_msgTypes[7]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ResultResponse.ProtoReflect.Descriptor instead.
|
||||
func (*ResultResponse) Descriptor() ([]byte, []int) {
|
||||
return file_agent_agent_proto_rawDescGZIP(), []int{7}
|
||||
}
|
||||
|
||||
func (x *ResultResponse) GetFile() []byte {
|
||||
if x != nil {
|
||||
return x.File
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_agent_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_agent_agent_proto_rawDesc = []byte{
|
||||
0x0a, 0x11, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x12, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x22, 0x2e, 0x0a, 0x0a, 0x52, 0x75,
|
||||
0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x70,
|
||||
0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63,
|
||||
0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2f, 0x0a, 0x0b, 0x52, 0x75,
|
||||
0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6d,
|
||||
0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
|
||||
0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2b, 0x0a, 0x0b, 0x41,
|
||||
0x6c, 0x67, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6c,
|
||||
0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x61,
|
||||
0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x30, 0x0a, 0x0c, 0x41, 0x6c, 0x67, 0x6f,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x61, 0x6c, 0x67, 0x6f,
|
||||
0x72, 0x69, 0x74, 0x68, 0x6d, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61,
|
||||
0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x49, 0x44, 0x22, 0x27, 0x0a, 0x0b, 0x44, 0x61,
|
||||
0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x61, 0x74,
|
||||
0x61, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x64, 0x61, 0x74, 0x61,
|
||||
0x73, 0x65, 0x74, 0x22, 0x2c, 0x0a, 0x0c, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x49, 0x44,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x49,
|
||||
0x44, 0x22, 0x0f, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x22, 0x24, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x0c, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x32, 0xdd, 0x01, 0x0a, 0x0c, 0x41, 0x67, 0x65,
|
||||
0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x52, 0x75, 0x6e,
|
||||
0x12, 0x11, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x75, 0x6e, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x31, 0x0a, 0x04, 0x41, 0x6c, 0x67,
|
||||
0x6f, 0x12, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x6c, 0x67, 0x6f, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x6c,
|
||||
0x67, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x31, 0x0a, 0x04,
|
||||
0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x61, 0x74,
|
||||
0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74,
|
||||
0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
|
||||
0x37, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x2e, 0x61, 0x67, 0x65, 0x6e,
|
||||
0x74, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x61, 0x67,
|
||||
0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
file_agent_agent_proto_rawDescOnce sync.Once
|
||||
file_agent_agent_proto_rawDescData = file_agent_agent_proto_rawDesc
|
||||
)
|
||||
|
||||
func file_agent_agent_proto_rawDescGZIP() []byte {
|
||||
file_agent_agent_proto_rawDescOnce.Do(func() {
|
||||
file_agent_agent_proto_rawDescData = protoimpl.X.CompressGZIP(file_agent_agent_proto_rawDescData)
|
||||
})
|
||||
return file_agent_agent_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_agent_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
|
||||
var file_agent_agent_proto_goTypes = []interface{}{
|
||||
(*RunRequest)(nil), // 0: agent.RunRequest
|
||||
(*RunResponse)(nil), // 1: agent.RunResponse
|
||||
(*AlgoRequest)(nil), // 2: agent.AlgoRequest
|
||||
(*AlgoResponse)(nil), // 3: agent.AlgoResponse
|
||||
(*DataRequest)(nil), // 4: agent.DataRequest
|
||||
(*DataResponse)(nil), // 5: agent.DataResponse
|
||||
(*ResultRequest)(nil), // 6: agent.ResultRequest
|
||||
(*ResultResponse)(nil), // 7: agent.ResultResponse
|
||||
}
|
||||
var file_agent_agent_proto_depIdxs = []int32{
|
||||
0, // 0: agent.AgentService.Run:input_type -> agent.RunRequest
|
||||
2, // 1: agent.AgentService.Algo:input_type -> agent.AlgoRequest
|
||||
4, // 2: agent.AgentService.Data:input_type -> agent.DataRequest
|
||||
6, // 3: agent.AgentService.Result:input_type -> agent.ResultRequest
|
||||
1, // 4: agent.AgentService.Run:output_type -> agent.RunResponse
|
||||
3, // 5: agent.AgentService.Algo:output_type -> agent.AlgoResponse
|
||||
5, // 6: agent.AgentService.Data:output_type -> agent.DataResponse
|
||||
7, // 7: agent.AgentService.Result:output_type -> agent.ResultResponse
|
||||
4, // [4:8] is the sub-list for method output_type
|
||||
0, // [0:4] is the sub-list for method input_type
|
||||
0, // [0:0] is the sub-list for extension type_name
|
||||
0, // [0:0] is the sub-list for extension extendee
|
||||
0, // [0:0] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_agent_agent_proto_init() }
|
||||
func file_agent_agent_proto_init() {
|
||||
if File_agent_agent_proto != nil {
|
||||
return
|
||||
}
|
||||
if !protoimpl.UnsafeEnabled {
|
||||
file_agent_agent_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*RunRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*RunResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*AlgoRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*AlgoResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*DataRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*DataResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*ResultRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_agent_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*ResultResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_agent_agent_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 8,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
GoTypes: file_agent_agent_proto_goTypes,
|
||||
DependencyIndexes: file_agent_agent_proto_depIdxs,
|
||||
MessageInfos: file_agent_agent_proto_msgTypes,
|
||||
}.Build()
|
||||
File_agent_agent_proto = out.File
|
||||
file_agent_agent_proto_rawDesc = nil
|
||||
file_agent_agent_proto_goTypes = nil
|
||||
file_agent_agent_proto_depIdxs = nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package agent;
|
||||
|
||||
option go_package = "./agent";
|
||||
|
||||
service AgentService {
|
||||
rpc Run(RunRequest) returns (RunResponse) {}
|
||||
rpc Algo(AlgoRequest) returns (AlgoResponse) {}
|
||||
rpc Data(DataRequest) returns (DataResponse) {}
|
||||
rpc Result(ResultRequest) returns (ResultResponse) {}
|
||||
}
|
||||
|
||||
message RunRequest { bytes computation = 1; }
|
||||
|
||||
message RunResponse { string Computation = 1; }
|
||||
|
||||
message AlgoRequest { bytes algorithm = 1; }
|
||||
|
||||
message AlgoResponse { string algorithmID = 1; }
|
||||
|
||||
message DataRequest { bytes dataset = 1; }
|
||||
|
||||
message DataResponse { string datasetID = 1; }
|
||||
|
||||
message ResultRequest {}
|
||||
|
||||
message ResultResponse { bytes file = 1; }
|
||||
@@ -0,0 +1,209 @@
|
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
context "context"
|
||||
grpc "google.golang.org/grpc"
|
||||
codes "google.golang.org/grpc/codes"
|
||||
status "google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// AgentServiceClient is the client API for AgentService service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
type AgentServiceClient interface {
|
||||
Run(ctx context.Context, in *RunRequest, opts ...grpc.CallOption) (*RunResponse, error)
|
||||
Algo(ctx context.Context, in *AlgoRequest, opts ...grpc.CallOption) (*AlgoResponse, error)
|
||||
Data(ctx context.Context, in *DataRequest, opts ...grpc.CallOption) (*DataResponse, error)
|
||||
Result(ctx context.Context, in *ResultRequest, opts ...grpc.CallOption) (*ResultResponse, error)
|
||||
}
|
||||
|
||||
type agentServiceClient struct {
|
||||
cc grpc.ClientConnInterface
|
||||
}
|
||||
|
||||
func NewAgentServiceClient(cc grpc.ClientConnInterface) AgentServiceClient {
|
||||
return &agentServiceClient{cc}
|
||||
}
|
||||
|
||||
func (c *agentServiceClient) Run(ctx context.Context, in *RunRequest, opts ...grpc.CallOption) (*RunResponse, error) {
|
||||
out := new(RunResponse)
|
||||
err := c.cc.Invoke(ctx, "/agent.AgentService/Run", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *agentServiceClient) Algo(ctx context.Context, in *AlgoRequest, opts ...grpc.CallOption) (*AlgoResponse, error) {
|
||||
out := new(AlgoResponse)
|
||||
err := c.cc.Invoke(ctx, "/agent.AgentService/Algo", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *agentServiceClient) Data(ctx context.Context, in *DataRequest, opts ...grpc.CallOption) (*DataResponse, error) {
|
||||
out := new(DataResponse)
|
||||
err := c.cc.Invoke(ctx, "/agent.AgentService/Data", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *agentServiceClient) Result(ctx context.Context, in *ResultRequest, opts ...grpc.CallOption) (*ResultResponse, error) {
|
||||
out := new(ResultResponse)
|
||||
err := c.cc.Invoke(ctx, "/agent.AgentService/Result", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AgentServiceServer is the server API for AgentService service.
|
||||
// All implementations must embed UnimplementedAgentServiceServer
|
||||
// for forward compatibility
|
||||
type AgentServiceServer interface {
|
||||
Run(context.Context, *RunRequest) (*RunResponse, error)
|
||||
Algo(context.Context, *AlgoRequest) (*AlgoResponse, error)
|
||||
Data(context.Context, *DataRequest) (*DataResponse, error)
|
||||
Result(context.Context, *ResultRequest) (*ResultResponse, error)
|
||||
mustEmbedUnimplementedAgentServiceServer()
|
||||
}
|
||||
|
||||
// UnimplementedAgentServiceServer must be embedded to have forward compatible implementations.
|
||||
type UnimplementedAgentServiceServer struct {
|
||||
}
|
||||
|
||||
func (UnimplementedAgentServiceServer) Run(context.Context, *RunRequest) (*RunResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Run not implemented")
|
||||
}
|
||||
func (UnimplementedAgentServiceServer) Algo(context.Context, *AlgoRequest) (*AlgoResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Algo not implemented")
|
||||
}
|
||||
func (UnimplementedAgentServiceServer) Data(context.Context, *DataRequest) (*DataResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Data not implemented")
|
||||
}
|
||||
func (UnimplementedAgentServiceServer) Result(context.Context, *ResultRequest) (*ResultResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Result not implemented")
|
||||
}
|
||||
func (UnimplementedAgentServiceServer) mustEmbedUnimplementedAgentServiceServer() {}
|
||||
|
||||
// UnsafeAgentServiceServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to AgentServiceServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeAgentServiceServer interface {
|
||||
mustEmbedUnimplementedAgentServiceServer()
|
||||
}
|
||||
|
||||
func RegisterAgentServiceServer(s grpc.ServiceRegistrar, srv AgentServiceServer) {
|
||||
s.RegisterService(&AgentService_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _AgentService_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RunRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(AgentServiceServer).Run(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/agent.AgentService/Run",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(AgentServiceServer).Run(ctx, req.(*RunRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _AgentService_Algo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(AlgoRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(AgentServiceServer).Algo(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/agent.AgentService/Algo",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(AgentServiceServer).Algo(ctx, req.(*AlgoRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _AgentService_Data_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(DataRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(AgentServiceServer).Data(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/agent.AgentService/Data",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(AgentServiceServer).Data(ctx, req.(*DataRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _AgentService_Result_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ResultRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(AgentServiceServer).Result(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/agent.AgentService/Result",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(AgentServiceServer).Result(ctx, req.(*ResultRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// AgentService_ServiceDesc is the grpc.ServiceDesc for AgentService service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var AgentService_ServiceDesc = grpc.ServiceDesc{
|
||||
ServiceName: "agent.AgentService",
|
||||
HandlerType: (*AgentServiceServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "Run",
|
||||
Handler: _AgentService_Run_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Algo",
|
||||
Handler: _AgentService_Algo_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Data",
|
||||
Handler: _AgentService_Data_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Result",
|
||||
Handler: _AgentService_Result_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
Metadata: "agent/agent.proto",
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package api contains API-related concerns: endpoint definitions, middlewares
|
||||
// and all resource representations.
|
||||
package api
|
||||
@@ -0,0 +1,214 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-kit/kit/endpoint"
|
||||
kitgrpc "github.com/go-kit/kit/transport/grpc"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
const svcName = "agent.AgentService"
|
||||
|
||||
type grpcClient struct {
|
||||
run endpoint.Endpoint
|
||||
algo endpoint.Endpoint
|
||||
data endpoint.Endpoint
|
||||
result endpoint.Endpoint
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// NewClient returns new gRPC client instance.
|
||||
func NewClient(conn *grpc.ClientConn, timeout time.Duration) agent.AgentServiceClient {
|
||||
return &grpcClient{
|
||||
run: kitgrpc.NewClient(
|
||||
conn,
|
||||
svcName,
|
||||
"Run",
|
||||
encodeRunRequest,
|
||||
decodeRunResponse,
|
||||
agent.RunResponse{},
|
||||
).Endpoint(),
|
||||
algo: kitgrpc.NewClient(
|
||||
conn,
|
||||
svcName,
|
||||
"Algo",
|
||||
encodeAlgoRequest,
|
||||
decodeAlgoResponse,
|
||||
agent.AlgoResponse{},
|
||||
).Endpoint(),
|
||||
data: kitgrpc.NewClient(
|
||||
conn,
|
||||
svcName,
|
||||
"Data",
|
||||
encodeDataRequest,
|
||||
decodeDataResponse,
|
||||
agent.DataResponse{},
|
||||
).Endpoint(),
|
||||
result: kitgrpc.NewClient(
|
||||
conn,
|
||||
svcName,
|
||||
"Result",
|
||||
encodeResultRequest,
|
||||
decodeResultResponse,
|
||||
agent.ResultResponse{},
|
||||
).Endpoint(),
|
||||
timeout: timeout,
|
||||
}
|
||||
}
|
||||
|
||||
// encodeRunRequest is a transport/grpc.EncodeRequestFunc that
|
||||
// converts a user-domain runReq to a gRPC request.
|
||||
func encodeRunRequest(_ context.Context, request interface{}) (interface{}, error) {
|
||||
req, ok := request.(*runReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid request type: %T", request)
|
||||
}
|
||||
|
||||
return &agent.RunRequest{
|
||||
Computation: req.Computation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// decodeRunResponse is a transport/grpc.DecodeResponseFunc that
|
||||
// converts a gRPC RunResponse to a user-domain response.
|
||||
func decodeRunResponse(_ context.Context, grpcResponse interface{}) (interface{}, error) {
|
||||
response, ok := grpcResponse.(*agent.RunResponse)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid response type: %T", grpcResponse)
|
||||
}
|
||||
return runRes{
|
||||
Computation: response.Computation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// encodeAlgoRequest is a transport/grpc.EncodeRequestFunc that
|
||||
// converts a user-domain algoReq to a gRPC request.
|
||||
func encodeAlgoRequest(_ context.Context, request interface{}) (interface{}, error) {
|
||||
req, ok := request.(*algoReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid request type: %T", request)
|
||||
}
|
||||
|
||||
return &agent.AlgoRequest{
|
||||
Algorithm: req.Algorithm,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// decodeAlgoResponse is a transport/grpc.DecodeResponseFunc that
|
||||
// converts a gRPC AlgoResponse to a user-domain response.
|
||||
func decodeAlgoResponse(_ context.Context, grpcResponse interface{}) (interface{}, error) {
|
||||
response, ok := grpcResponse.(*agent.AlgoResponse)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid response type: %T", grpcResponse)
|
||||
}
|
||||
|
||||
return algoRes{
|
||||
AlgorithmID: response.AlgorithmID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// encodeDataRequest is a transport/grpc.EncodeRequestFunc that
|
||||
// converts a user-domain dataReq to a gRPC request.
|
||||
func encodeDataRequest(_ context.Context, request interface{}) (interface{}, error) {
|
||||
req, ok := request.(*dataReq)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid request type: %T", request)
|
||||
}
|
||||
|
||||
return &agent.DataRequest{
|
||||
Dataset: req.Dataset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// decodeDataResponse is a transport/grpc.DecodeResponseFunc that
|
||||
// converts a gRPC DataResponse to a user-domain response.
|
||||
func decodeDataResponse(_ context.Context, grpcResponse interface{}) (interface{}, error) {
|
||||
response, ok := grpcResponse.(*agent.DataResponse)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid response type: %T", grpcResponse)
|
||||
}
|
||||
|
||||
return dataRes{
|
||||
DatasetID: response.DatasetID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// encodeResultRequest is a transport/grpc.EncodeRequestFunc that
|
||||
// converts a user-domain resultReq to a gRPC request.
|
||||
func encodeResultRequest(_ context.Context, request interface{}) (interface{}, error) {
|
||||
// No request parameters needed for retrieving computation result file
|
||||
return &agent.ResultRequest{}, nil
|
||||
}
|
||||
|
||||
// decodeResultResponse is a transport/grpc.DecodeResponseFunc that
|
||||
// converts a gRPC ResultResponse to a user-domain response.
|
||||
func decodeResultResponse(_ context.Context, grpcResponse interface{}) (interface{}, error) {
|
||||
response, ok := grpcResponse.(*agent.ResultResponse)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid response type: %T", grpcResponse)
|
||||
}
|
||||
|
||||
return resultRes{
|
||||
File: response.File,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run implements the Run method of the agent.AgentServiceClient interface.
|
||||
func (c grpcClient) Run(ctx context.Context, request *agent.RunRequest, _ ...grpc.CallOption) (*agent.RunResponse, error) {
|
||||
ctx, close := context.WithTimeout(ctx, c.timeout)
|
||||
defer close()
|
||||
|
||||
res, err := c.run(ctx, &runReq{Computation: request.Computation})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
runRes := res.(runRes)
|
||||
return &agent.RunResponse{Computation: runRes.Computation}, nil
|
||||
}
|
||||
|
||||
// Algo implements the Algo method of the agent.AgentServiceClient interface.
|
||||
func (c grpcClient) Algo(ctx context.Context, request *agent.AlgoRequest, _ ...grpc.CallOption) (*agent.AlgoResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
res, err := c.algo(ctx, &algoReq{Algorithm: request.Algorithm})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
algoRes := res.(algoRes)
|
||||
return &agent.AlgoResponse{AlgorithmID: algoRes.AlgorithmID}, nil
|
||||
}
|
||||
|
||||
// Data implements the Data method of the agent.AgentServiceClient interface.
|
||||
func (c grpcClient) Data(ctx context.Context, request *agent.DataRequest, _ ...grpc.CallOption) (*agent.DataResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
res, err := c.data(ctx, &dataReq{Dataset: request.Dataset})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dataRes := res.(dataRes)
|
||||
return &agent.DataResponse{DatasetID: dataRes.DatasetID}, nil
|
||||
}
|
||||
|
||||
// Result implements the Result method of the agent.AgentServiceClient interface.
|
||||
func (c grpcClient) Result(ctx context.Context, request *agent.ResultRequest, _ ...grpc.CallOption) (*agent.ResultResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
res, err := c.result(ctx, &resultReq{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resultRes := res.(resultRes)
|
||||
return &agent.ResultResponse{File: resultRes.File}, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package grpc contains implementation of kit service gRPC API.
|
||||
package grpc
|
||||
@@ -0,0 +1,82 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/go-kit/kit/endpoint"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
func runEndpoint(svc agent.Service) endpoint.Endpoint {
|
||||
return func(ctx context.Context, request interface{}) (interface{}, error) {
|
||||
req := request.(runReq)
|
||||
|
||||
if err := req.validate(); err != nil {
|
||||
return runRes{}, err
|
||||
}
|
||||
|
||||
var computation agent.Computation
|
||||
err := json.Unmarshal(req.Computation, &computation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
computationStr, err := svc.Run(ctx, computation)
|
||||
if err != nil {
|
||||
return runRes{}, err
|
||||
}
|
||||
|
||||
return runRes{Computation: computationStr}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func algoEndpoint(svc agent.Service) endpoint.Endpoint {
|
||||
return func(ctx context.Context, request interface{}) (interface{}, error) {
|
||||
req := request.(algoReq)
|
||||
|
||||
if err := req.validate(); err != nil {
|
||||
return algoRes{}, err
|
||||
}
|
||||
|
||||
algorithmID, err := svc.Algo(ctx, req.Algorithm)
|
||||
if err != nil {
|
||||
return algoRes{}, err
|
||||
}
|
||||
|
||||
return algoRes{AlgorithmID: algorithmID}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func dataEndpoint(svc agent.Service) endpoint.Endpoint {
|
||||
return func(ctx context.Context, request interface{}) (interface{}, error) {
|
||||
req := request.(dataReq)
|
||||
|
||||
if err := req.validate(); err != nil {
|
||||
return dataRes{}, err
|
||||
}
|
||||
|
||||
datasetID, err := svc.Data(ctx, req.Dataset)
|
||||
if err != nil {
|
||||
return dataRes{}, err
|
||||
}
|
||||
|
||||
return dataRes{DatasetID: datasetID}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func resultEndpoint(svc agent.Service) endpoint.Endpoint {
|
||||
return func(ctx context.Context, request interface{}) (interface{}, error) {
|
||||
req := request.(resultReq)
|
||||
|
||||
if err := req.validate(); err != nil {
|
||||
return resultRes{}, err
|
||||
}
|
||||
file, err := svc.Result(ctx)
|
||||
if err != nil {
|
||||
return resultRes{}, err
|
||||
}
|
||||
|
||||
return resultRes{File: file}, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package grpc
|
||||
|
||||
import "errors"
|
||||
|
||||
type runReq struct {
|
||||
Computation []byte `protobuf:"bytes,1,opt,name=algorithm,proto3" json:"algorithm,omitempty"`
|
||||
}
|
||||
|
||||
func (req runReq) validate() error {
|
||||
if len(req.Computation) == 0 {
|
||||
return errors.New("algorithm binary is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type algoReq struct {
|
||||
Algorithm []byte `protobuf:"bytes,1,opt,name=algorithm,proto3" json:"algorithm,omitempty"`
|
||||
}
|
||||
|
||||
func (req algoReq) validate() error {
|
||||
if len(req.Algorithm) == 0 {
|
||||
return errors.New("algorithm binary is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type dataReq struct {
|
||||
Dataset []byte `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"`
|
||||
}
|
||||
|
||||
func (req dataReq) validate() error {
|
||||
if len(req.Dataset) == 0 {
|
||||
return errors.New("dataset CSV file is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type resultReq struct {
|
||||
// No request parameters needed for retrieving computation result file
|
||||
}
|
||||
|
||||
func (req resultReq) validate() error {
|
||||
// No request parameters to validate, so no validation logic needed
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package grpc
|
||||
|
||||
type runRes struct {
|
||||
Computation string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type algoRes struct {
|
||||
AlgorithmID string `json:"algorithmId,omitempty"`
|
||||
}
|
||||
|
||||
type dataRes struct {
|
||||
DatasetID string `json:"datasetId,omitempty"`
|
||||
}
|
||||
|
||||
type resultRes struct {
|
||||
File []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
kitgrpc "github.com/go-kit/kit/transport/grpc"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
type grpcServer struct {
|
||||
run kitgrpc.Handler
|
||||
algo kitgrpc.Handler
|
||||
data kitgrpc.Handler
|
||||
result kitgrpc.Handler
|
||||
agent.UnimplementedAgentServiceServer
|
||||
}
|
||||
|
||||
// NewServer returns new AgentServiceServer instance.
|
||||
func NewServer(svc agent.Service) agent.AgentServiceServer {
|
||||
return &grpcServer{
|
||||
run: kitgrpc.NewServer(
|
||||
runEndpoint(svc),
|
||||
decodeRunRequest,
|
||||
encodeRunResponse,
|
||||
),
|
||||
algo: kitgrpc.NewServer(
|
||||
algoEndpoint(svc),
|
||||
decodeAlgoRequest,
|
||||
encodeAlgoResponse,
|
||||
),
|
||||
data: kitgrpc.NewServer(
|
||||
dataEndpoint(svc),
|
||||
decodeDataRequest,
|
||||
encodeDataResponse,
|
||||
),
|
||||
result: kitgrpc.NewServer(
|
||||
resultEndpoint(svc),
|
||||
decodeResultRequest,
|
||||
encodeResultResponse,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
func decodeRunRequest(_ context.Context, grpcReq interface{}) (interface{}, error) {
|
||||
req := grpcReq.(*agent.RunRequest)
|
||||
|
||||
return runReq{
|
||||
Computation: req.Computation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeRunResponse(_ context.Context, response interface{}) (interface{}, error) {
|
||||
res := response.(runRes)
|
||||
return &agent.RunResponse{
|
||||
Computation: res.Computation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeAlgoRequest(_ context.Context, grpcReq interface{}) (interface{}, error) {
|
||||
req := grpcReq.(*agent.AlgoRequest)
|
||||
|
||||
return algoReq{
|
||||
Algorithm: req.Algorithm,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeAlgoResponse(_ context.Context, response interface{}) (interface{}, error) {
|
||||
res := response.(algoRes)
|
||||
return &agent.AlgoResponse{
|
||||
AlgorithmID: res.AlgorithmID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeDataRequest(_ context.Context, grpcReq interface{}) (interface{}, error) {
|
||||
req := grpcReq.(*agent.DataRequest)
|
||||
|
||||
return dataReq{
|
||||
Dataset: req.Dataset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func encodeDataResponse(_ context.Context, response interface{}) (interface{}, error) {
|
||||
res := response.(dataRes)
|
||||
return &agent.DataResponse{
|
||||
DatasetID: res.DatasetID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeResultRequest(_ context.Context, grpcReq interface{}) (interface{}, error) {
|
||||
// No fields to extract from gRPC request, so returning an empty struct
|
||||
return resultReq{}, nil
|
||||
}
|
||||
|
||||
func encodeResultResponse(_ context.Context, response interface{}) (interface{}, error) {
|
||||
res := response.(resultRes)
|
||||
return &agent.ResultResponse{
|
||||
File: res.File,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *grpcServer) Run(ctx context.Context, req *agent.RunRequest) (*agent.RunResponse, error) {
|
||||
_, res, err := s.run.ServeGRPC(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rr := res.(*agent.RunResponse)
|
||||
return rr, nil
|
||||
}
|
||||
|
||||
func (s *grpcServer) Algo(ctx context.Context, req *agent.AlgoRequest) (*agent.AlgoResponse, error) {
|
||||
_, res, err := s.algo.ServeGRPC(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ar := res.(*agent.AlgoResponse)
|
||||
return ar, nil
|
||||
}
|
||||
|
||||
func (s *grpcServer) Data(ctx context.Context, req *agent.DataRequest) (*agent.DataResponse, error) {
|
||||
_, res, err := s.data.ServeGRPC(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dr := res.(*agent.DataResponse)
|
||||
return dr, nil
|
||||
}
|
||||
|
||||
func (s *grpcServer) Result(ctx context.Context, req *agent.ResultRequest) (*agent.ResultResponse, error) {
|
||||
_, res, err := s.result.ServeGRPC(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rr := res.(*agent.ResultResponse)
|
||||
return rr, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package http contains implementation of kit service HTTP API.
|
||||
package http
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/go-kit/kit/endpoint"
|
||||
agent "github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
func runEndpoint(svc agent.Service) endpoint.Endpoint {
|
||||
return func(ctx context.Context, request interface{}) (interface{}, error) {
|
||||
req := request.(runReq)
|
||||
|
||||
if err := req.validate(); err != nil {
|
||||
return runRes{}, err
|
||||
}
|
||||
cmp := agent.Computation{
|
||||
ID: req.computation.ID,
|
||||
Name: req.computation.Name,
|
||||
Description: req.computation.Description,
|
||||
Status: req.computation.Status,
|
||||
Owner: req.computation.Owner,
|
||||
StartTime: req.computation.StartTime,
|
||||
EndTime: req.computation.EndTime,
|
||||
Datasets: req.computation.Datasets,
|
||||
Algorithms: req.computation.Algorithms,
|
||||
DatasetProviders: req.computation.DatasetProviders,
|
||||
AlgorithmProviders: req.computation.AlgorithmProviders,
|
||||
ResultConsumers: req.computation.ResultConsumers,
|
||||
Ttl: req.computation.Ttl,
|
||||
Metadata: req.computation.Metadata,
|
||||
}
|
||||
|
||||
cmpStr, err := svc.Run(ctx, cmp)
|
||||
if err != nil {
|
||||
return runRes{}, err
|
||||
}
|
||||
|
||||
return runRes{Computation: cmpStr}, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package http
|
||||
|
||||
import agent "github.com/ultravioletrs/agent/agent"
|
||||
|
||||
type runReq struct {
|
||||
computation agent.Computation
|
||||
}
|
||||
|
||||
func (req runReq) validate() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mainflux/mainflux"
|
||||
)
|
||||
|
||||
var _ mainflux.Response = (*runRes)(nil)
|
||||
|
||||
type runRes struct {
|
||||
Computation string `json:"computation"`
|
||||
}
|
||||
|
||||
func (res runRes) Code() int {
|
||||
return http.StatusOK
|
||||
}
|
||||
|
||||
func (res runRes) Headers() map[string]string {
|
||||
return map[string]string{}
|
||||
}
|
||||
|
||||
func (res runRes) Empty() bool {
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
kithttp "github.com/go-kit/kit/transport/http"
|
||||
"github.com/go-zoo/bone"
|
||||
"github.com/mainflux/mainflux"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
|
||||
const contentType = "application/json"
|
||||
|
||||
var (
|
||||
errUnsupportedContentType = errors.New("unsupported content type")
|
||||
errInvalidQueryParams = errors.New("invalid query params")
|
||||
)
|
||||
|
||||
// MakeHandler returns a HTTP handler for API endpoints.
|
||||
func MakeHandler(svc agent.Service, instanceID string) http.Handler {
|
||||
opts := []kithttp.ServerOption{
|
||||
kithttp.ServerErrorEncoder(encodeError),
|
||||
}
|
||||
|
||||
r := bone.New()
|
||||
|
||||
r.Post("/run", otelhttp.NewHandler(kithttp.NewServer(
|
||||
runEndpoint(svc),
|
||||
decodeRun,
|
||||
encodeResponse,
|
||||
opts...,
|
||||
), "run"))
|
||||
|
||||
r.GetFunc("/health", mainflux.Health("agent", instanceID))
|
||||
r.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func decodeRun(_ context.Context, r *http.Request) (interface{}, error) {
|
||||
if !strings.Contains(r.Header.Get("Content-Type"), contentType) {
|
||||
return nil, errUnsupportedContentType
|
||||
}
|
||||
|
||||
var req runReq
|
||||
if err := json.NewDecoder(r.Body).Decode(&req.computation); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func encodeResponse(_ context.Context, w http.ResponseWriter, response interface{}) error {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
|
||||
if ar, ok := response.(mainflux.Response); ok {
|
||||
for k, v := range ar.Headers() {
|
||||
w.Header().Set(k, v)
|
||||
}
|
||||
|
||||
w.WriteHeader(ar.Code())
|
||||
|
||||
if ar.Empty() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
||||
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
|
||||
switch err {
|
||||
case agent.ErrMalformedEntity:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case agent.ErrUnauthorizedAccess:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
case errUnsupportedContentType:
|
||||
w.WriteHeader(http.StatusUnsupportedMediaType)
|
||||
case errInvalidQueryParams:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case io.ErrUnexpectedEOF:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case io.EOF:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
default:
|
||||
switch err.(type) {
|
||||
case *json.SyntaxError:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case *json.UnmarshalTypeError:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
default:
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//go:build !test
|
||||
// +build !test
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
log "github.com/mainflux/mainflux/logger"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
var _ agent.Service = (*loggingMiddleware)(nil)
|
||||
|
||||
type loggingMiddleware struct {
|
||||
logger log.Logger
|
||||
svc agent.Service
|
||||
}
|
||||
|
||||
// LoggingMiddleware adds logging facilities to the core service.
|
||||
func LoggingMiddleware(svc agent.Service, logger log.Logger) agent.Service {
|
||||
return &loggingMiddleware{logger, svc}
|
||||
}
|
||||
|
||||
func (lm *loggingMiddleware) Run(ctx context.Context, cmp agent.Computation) (response string, err error) {
|
||||
defer func(begin time.Time) {
|
||||
message := fmt.Sprintf("Method Run for computation %s took %s to complete", cmp.ID, time.Since(begin))
|
||||
if err != nil {
|
||||
lm.logger.Warn(fmt.Sprintf("%s with error: %s.", message, err))
|
||||
return
|
||||
}
|
||||
lm.logger.Info(fmt.Sprintf("%s without errors.", message))
|
||||
}(time.Now())
|
||||
|
||||
return lm.svc.Run(ctx, cmp)
|
||||
}
|
||||
|
||||
func (lm *loggingMiddleware) Algo(ctx context.Context, algorithm []byte) (response string, err error) {
|
||||
defer func(begin time.Time) {
|
||||
message := fmt.Sprintf("Method Algo took %s to complete", time.Since(begin))
|
||||
if err != nil {
|
||||
lm.logger.Warn(fmt.Sprintf("%s with error: %s", message, err))
|
||||
return
|
||||
}
|
||||
lm.logger.Info(fmt.Sprintf("%s without errors", message))
|
||||
}(time.Now())
|
||||
|
||||
return lm.svc.Algo(ctx, algorithm)
|
||||
}
|
||||
|
||||
func (lm *loggingMiddleware) Data(ctx context.Context, dataset []byte) (response string, err error) {
|
||||
defer func(begin time.Time) {
|
||||
message := fmt.Sprintf("Method Data took %s to complete", time.Since(begin))
|
||||
if err != nil {
|
||||
lm.logger.Warn(fmt.Sprintf("%s with error: %s", message, err))
|
||||
return
|
||||
}
|
||||
lm.logger.Info(fmt.Sprintf("%s without errors", message))
|
||||
}(time.Now())
|
||||
|
||||
return lm.svc.Data(ctx, dataset)
|
||||
}
|
||||
|
||||
func (lm *loggingMiddleware) Result(ctx context.Context) (response []byte, err error) {
|
||||
defer func(begin time.Time) {
|
||||
message := fmt.Sprintf("Method Result took %s to complete", time.Since(begin))
|
||||
if err != nil {
|
||||
lm.logger.Warn(fmt.Sprintf("%s with error: %s", message, err))
|
||||
return
|
||||
}
|
||||
lm.logger.Info(fmt.Sprintf("%s without errors", message))
|
||||
}(time.Now())
|
||||
|
||||
return lm.svc.Result(ctx)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//go:build !test
|
||||
// +build !test
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
var _ agent.Service = (*metricsMiddleware)(nil)
|
||||
|
||||
type metricsMiddleware struct {
|
||||
counter metrics.Counter
|
||||
latency metrics.Histogram
|
||||
svc agent.Service
|
||||
}
|
||||
|
||||
// MetricsMiddleware instruments core service by tracking request count and
|
||||
// latency.
|
||||
func MetricsMiddleware(svc agent.Service, counter metrics.Counter, latency metrics.Histogram) agent.Service {
|
||||
return &metricsMiddleware{
|
||||
counter: counter,
|
||||
latency: latency,
|
||||
svc: svc,
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *metricsMiddleware) Run(ctx context.Context, cmp agent.Computation) (string, error) {
|
||||
defer func(begin time.Time) {
|
||||
ms.counter.With("method", "run").Add(1)
|
||||
ms.latency.With("method", "run").Observe(time.Since(begin).Seconds())
|
||||
}(time.Now())
|
||||
|
||||
return ms.svc.Run(ctx, cmp)
|
||||
}
|
||||
|
||||
func (ms *metricsMiddleware) Algo(ctx context.Context, algorithm []byte) (string, error) {
|
||||
defer func(begin time.Time) {
|
||||
ms.counter.With("method", "algo").Add(1)
|
||||
ms.latency.With("method", "algo").Observe(time.Since(begin).Seconds())
|
||||
}(time.Now())
|
||||
|
||||
return ms.svc.Algo(ctx, algorithm)
|
||||
}
|
||||
|
||||
func (ms *metricsMiddleware) Data(ctx context.Context, dataset []byte) (string, error) {
|
||||
defer func(begin time.Time) {
|
||||
ms.counter.With("method", "data").Add(1)
|
||||
ms.latency.With("method", "data").Observe(time.Since(begin).Seconds())
|
||||
}(time.Now())
|
||||
|
||||
return ms.svc.Data(ctx, dataset)
|
||||
}
|
||||
|
||||
func (ms *metricsMiddleware) Result(ctx context.Context) ([]byte, error) {
|
||||
defer func(begin time.Time) {
|
||||
ms.counter.With("method", "result").Add(1)
|
||||
ms.latency.With("method", "result").Observe(time.Since(begin).Seconds())
|
||||
}(time.Now())
|
||||
|
||||
return ms.svc.Result(ctx)
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package agent
|
||||
|
||||
import "time"
|
||||
|
||||
type Computation struct {
|
||||
ID string `json:"id,omitempty" db:"id"`
|
||||
Name string `json:"name,omitempty" db:"name"`
|
||||
Description string `json:"description,omitempty" db:"description"`
|
||||
Status string `json:"status,omitempty" db:"status"`
|
||||
Owner string `json:"owner,omitempty" db:"owner"`
|
||||
StartTime time.Time `json:"start_time,omitempty" db:"start_time"`
|
||||
EndTime time.Time `json:"end_time,omitempty" db:"end_time"`
|
||||
Datasets []string `json:"datasets,omitempty" db:"datasets"`
|
||||
Algorithms []string `json:"algorithms,omitempty" db:"algorithms"`
|
||||
DatasetProviders []string `json:"dataset_providers,omitempty" db:"dataset_providers"`
|
||||
AlgorithmProviders []string `json:"algorithm_providers,omitempty" db:"algorithm_providers"`
|
||||
ResultConsumers []string `json:"result_consumers,omitempty" db:"result_consumers"`
|
||||
Ttl int32 `json:"ttl,omitempty" db:"ttl"`
|
||||
Metadata Metadata `json:"metadata,omitempty" db:"metadata"`
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package things contains the domain concept definitions needed to support
|
||||
// Mainflux agent service functionality.
|
||||
|
||||
package agent
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
|
||||
socket "github.com/ultravioletrs/agent/pkg"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMalformedEntity indicates malformed entity specification (e.g.
|
||||
// invalid username or password).
|
||||
ErrMalformedEntity = errors.New("malformed entity specification")
|
||||
|
||||
// ErrUnauthorizedAccess indicates missing or invalid credentials provided
|
||||
// when accessing a protected resource.
|
||||
ErrUnauthorizedAccess = errors.New("missing or invalid credentials provided")
|
||||
)
|
||||
|
||||
type Metadata map[string]interface{}
|
||||
|
||||
// Service specifies an API that must be fullfiled by the domain service
|
||||
// implementation, and all of its decorators (e.g. logging & metrics).
|
||||
type Service interface {
|
||||
Run(ctx context.Context, cmp Computation) (string, error)
|
||||
Algo(ctx context.Context, algorithm []byte) (string, error)
|
||||
Data(ctx context.Context, dataset []byte) (string, error)
|
||||
Result(ctx context.Context) ([]byte, error)
|
||||
}
|
||||
|
||||
type agentService struct {
|
||||
computation Computation
|
||||
algorithms [][]byte
|
||||
datasets [][]byte
|
||||
result []byte
|
||||
}
|
||||
|
||||
const socketPath = "unix_socket"
|
||||
const pyRuntime = "python3"
|
||||
|
||||
var _ Service = (*agentService)(nil)
|
||||
|
||||
// New instantiates the agent service implementation.
|
||||
func New() Service {
|
||||
return &agentService{}
|
||||
}
|
||||
|
||||
func (as *agentService) Run(ctx context.Context, cmp Computation) (string, error) {
|
||||
cmpJSON, err := json.Marshal(cmp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
as.computation = cmp
|
||||
|
||||
return string(cmpJSON), nil // return the JSON string as the function's string return value
|
||||
}
|
||||
|
||||
func (as *agentService) Algo(ctx context.Context, algorithm []byte) (string, error) {
|
||||
// Implement the logic for the Algo method based on your requirements
|
||||
// Use the provided ctx and algorithm parameters as needed
|
||||
|
||||
as.algorithms = append(as.algorithms, algorithm)
|
||||
|
||||
// Perform some processing on the algorithm byte array
|
||||
// For example, generate a unique ID for the algorithm
|
||||
algorithmID := "algo123"
|
||||
|
||||
// Return the algorithm ID or an error
|
||||
return algorithmID, nil
|
||||
}
|
||||
|
||||
func (as *agentService) Data(ctx context.Context, dataset []byte) (string, error) {
|
||||
// Implement the logic for the Data method based on your requirements
|
||||
// Use the provided ctx and dataset parameters as needed
|
||||
|
||||
as.datasets = append(as.datasets, dataset)
|
||||
|
||||
// Perform some processing on the dataset string
|
||||
// For example, generate a unique ID for the dataset
|
||||
datasetID := "dataset456"
|
||||
|
||||
// Return the dataset ID or an error
|
||||
return datasetID, nil
|
||||
}
|
||||
|
||||
func (as *agentService) Result(ctx context.Context) ([]byte, error) {
|
||||
// Implement the logic for the Result method based on your requirements
|
||||
// Use the provided ctx parameter as needed
|
||||
|
||||
result, err := run(as.algorithms[0], as.datasets[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error performing computation: %v", err)
|
||||
}
|
||||
as.result = result
|
||||
|
||||
// Return the result file or an error
|
||||
return as.result, nil
|
||||
}
|
||||
|
||||
func run(algoContent []byte, dataContent []byte) ([]byte, error) {
|
||||
listener, err := socket.StartUnixSocketServer(socketPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating stdout pipe: %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
// Create channels for received data and errors
|
||||
dataChannel := make(chan []byte)
|
||||
errorChannel := make(chan error)
|
||||
go socket.AcceptConnection(listener, dataChannel, errorChannel)
|
||||
|
||||
// Construct the Python script content with CSV data as a command-line argument
|
||||
script := string(algoContent)
|
||||
data := string(dataContent)
|
||||
cmd := exec.Command(pyRuntime, "-c", script, data, socketPath)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("error starting Python script: %v", err)
|
||||
}
|
||||
|
||||
var receivedData []byte
|
||||
select {
|
||||
case receivedData = <-dataChannel:
|
||||
case err = <-errorChannel:
|
||||
return nil, fmt.Errorf("error receiving data: %v", err)
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return nil, fmt.Errorf("python script execution error: %v", err)
|
||||
}
|
||||
|
||||
return receivedData, nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Package tracing provides tracing instrumentation for cocos auth service.
|
||||
//
|
||||
// This package provides tracing middleware for cocos auth service.
|
||||
// It can be used to trace incoming requests and add tracing capabilities to
|
||||
// cocos auth service.
|
||||
//
|
||||
// For more details about tracing instrumentation refer
|
||||
// to the documentation at https://docs.mainflux.io/tracing/.
|
||||
package tracing
|
||||
@@ -0,0 +1,59 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var _ agent.Service = (*tracingMiddleware)(nil)
|
||||
|
||||
type tracingMiddleware struct {
|
||||
tracer trace.Tracer
|
||||
svc agent.Service
|
||||
}
|
||||
|
||||
// New returns a new auth service with tracing capabilities.
|
||||
func New(svc agent.Service, tracer trace.Tracer) agent.Service {
|
||||
return &tracingMiddleware{tracer, svc}
|
||||
}
|
||||
|
||||
func (tm *tracingMiddleware) Run(ctx context.Context, cmp agent.Computation) (string, error) {
|
||||
ctx, span := tm.tracer.Start(ctx, "run", trace.WithAttributes(
|
||||
attribute.String("id", cmp.ID),
|
||||
attribute.String("name", cmp.Name),
|
||||
attribute.String("description", cmp.Description),
|
||||
attribute.String("status", cmp.Status),
|
||||
attribute.String("start_time", cmp.StartTime.String()),
|
||||
attribute.String("end_time", cmp.EndTime.String()),
|
||||
attribute.StringSlice("dataset_providers", cmp.DatasetProviders),
|
||||
attribute.StringSlice("algorithm_providers", cmp.AlgorithmProviders),
|
||||
attribute.StringSlice("result_consumers", cmp.ResultConsumers),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
return tm.svc.Run(ctx, cmp)
|
||||
}
|
||||
|
||||
func (tm *tracingMiddleware) Algo(ctx context.Context, algorithm []byte) (string, error) {
|
||||
ctx, span := tm.tracer.Start(ctx, "algo")
|
||||
defer span.End()
|
||||
|
||||
return tm.svc.Algo(ctx, algorithm)
|
||||
}
|
||||
|
||||
func (tm *tracingMiddleware) Data(ctx context.Context, dataset []byte) (string, error) {
|
||||
ctx, span := tm.tracer.Start(ctx, "data")
|
||||
defer span.End()
|
||||
|
||||
return tm.svc.Data(ctx, dataset)
|
||||
}
|
||||
|
||||
func (tm *tracingMiddleware) Result(ctx context.Context) ([]byte, error) {
|
||||
ctx, span := tm.tracer.Start(ctx, "result")
|
||||
defer span.End()
|
||||
|
||||
return tm.svc.Result(ctx)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# Agent CLI
|
||||
|
||||
This repository contains the command-line interface (CLI) tool for interacting with the Agent service. The CLI allows you to perform various tasks such as running computations, uploading algorithms and datasets, and retrieving results.
|
||||
|
||||
## Build
|
||||
|
||||
From the project root:
|
||||
|
||||
```bash
|
||||
make cli
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
#### Run Computation
|
||||
|
||||
To run a computation, use the following command:
|
||||
|
||||
```bash
|
||||
./build/cocos-cli run --computation '{"name": "my-computation"}'
|
||||
```
|
||||
|
||||
#### Upload Algorithm
|
||||
|
||||
To upload an algorithm, use the following command:
|
||||
|
||||
```bash
|
||||
./build/cocos-cli algo /path/to/algorithm
|
||||
```
|
||||
|
||||
#### Upload Dataset
|
||||
|
||||
To upload a dataset, use the following command:
|
||||
|
||||
```bash
|
||||
./build/cocos-cli data /path/to/dataset.csv
|
||||
```
|
||||
|
||||
#### Retrieve result
|
||||
|
||||
To retrieve the computation result, use the following command:
|
||||
|
||||
```bash
|
||||
./build/cocos-cli result
|
||||
```
|
||||
|
||||
## Installtion
|
||||
|
||||
To use the CLI, you have the option to install it globally on your system. Here's how:
|
||||
|
||||
### Build the CLI:
|
||||
|
||||
Navigate to the project root and run the following command to build the CLI binary:
|
||||
|
||||
```bash
|
||||
make install-cli
|
||||
```
|
||||
@@ -0,0 +1,37 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
agentsdk "github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
func NewAlgorithmsCmd(sdk agentsdk.SDK) *cobra.Command {
|
||||
|
||||
return &cobra.Command{
|
||||
Use: "algo",
|
||||
Short: "Upload an algorithm binary",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
algorithmFile := args[0]
|
||||
|
||||
log.Println("Uploading algorithm binary:", algorithmFile)
|
||||
|
||||
algorithm, err := os.ReadFile(algorithmFile)
|
||||
if err != nil {
|
||||
log.Println("Error reading dataset file:", err)
|
||||
return
|
||||
}
|
||||
|
||||
response, err := sdk.UploadAlgorithm(algorithm)
|
||||
if err != nil {
|
||||
log.Println("Error uploading algorithm:", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Succesfully uploaded algorithm:", response)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
agentsdk "github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
func NewDatasetsCmd(sdk agentsdk.SDK) *cobra.Command {
|
||||
|
||||
return &cobra.Command{
|
||||
Use: "data",
|
||||
Short: "Upload a dataset CSV file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
datasetFile := args[0]
|
||||
|
||||
log.Println("Uploading dataset CSV:", datasetFile)
|
||||
|
||||
dataset, err := os.ReadFile(datasetFile)
|
||||
if err != nil {
|
||||
log.Println("Error reading dataset file:", err)
|
||||
return
|
||||
}
|
||||
|
||||
response, err := sdk.UploadDataset(dataset)
|
||||
if err != nil {
|
||||
log.Println("Error uploading dataset:", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Response:", response)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: Computation Service API
|
||||
version: 1.0.0
|
||||
servers:
|
||||
- url: https://api.example.com/v1
|
||||
paths:
|
||||
/run:
|
||||
post:
|
||||
summary: Run a computation
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
description: The computation binary file (Linux executable)
|
||||
responses:
|
||||
"200":
|
||||
description: Computation started
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
computationId:
|
||||
type: string
|
||||
description: Identifier for the computation
|
||||
|
||||
/algo:
|
||||
post:
|
||||
summary: Upload algorithm binary
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
description: The algorithm binary file (Linux executable)
|
||||
responses:
|
||||
"200":
|
||||
description: Algorithm binary uploaded
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
algorithmId:
|
||||
type: string
|
||||
description: Identifier for the uploaded algorithm binary
|
||||
|
||||
/data:
|
||||
post:
|
||||
summary: Upload dataset CSV file
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
description: The dataset CSV file as a binary
|
||||
responses:
|
||||
"200":
|
||||
description: Dataset CSV uploaded
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
datasetId:
|
||||
type: string
|
||||
description: Identifier for the uploaded dataset CSV
|
||||
|
||||
/result:
|
||||
get:
|
||||
summary: Retrieve computation result file
|
||||
responses:
|
||||
"200":
|
||||
description: Computation result file retrieved successfully
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
description: The computation result file
|
||||
@@ -0,0 +1,36 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
agentsdk "github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
const resultFilePath = "result.bin"
|
||||
|
||||
func NewResultsCmd(sdk agentsdk.SDK) *cobra.Command {
|
||||
|
||||
return &cobra.Command{
|
||||
Use: "result",
|
||||
Short: "Retrieve computation result file",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
log.Println("Retrieving computation result file")
|
||||
|
||||
result, err := sdk.Result()
|
||||
if err != nil {
|
||||
log.Println("Error retrieving computation result:", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = os.WriteFile(resultFilePath, result, 0644)
|
||||
if err != nil {
|
||||
log.Println("Error saving computation result:", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Computation result retrieved and saved successfully!")
|
||||
},
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
agentsdk "github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
func NewRunCmd(sdk agentsdk.SDK) *cobra.Command {
|
||||
var computationJSON string
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "Run a computation",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
log.Println("Running computation")
|
||||
|
||||
var computation agentsdk.Computation
|
||||
if err := json.Unmarshal([]byte(computationJSON), &computation); err != nil {
|
||||
log.Println("Failed to unmarshal computation JSON:", err)
|
||||
return
|
||||
}
|
||||
|
||||
response, err := sdk.Run(computation)
|
||||
if err != nil {
|
||||
log.Println("Error running computation:", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Response:", response)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&computationJSON, "computation", "", "JSON representation of the computation")
|
||||
|
||||
if err := cmd.MarkFlagRequired("computation"); err != nil {
|
||||
log.Fatalf("Failed to mark flag as required: %s", err)
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
func SetSDK(s sdk.SDK) {
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
mflog "github.com/mainflux/mainflux/logger"
|
||||
"github.com/mainflux/mainflux/pkg/uuid"
|
||||
agent "github.com/ultravioletrs/agent/agent"
|
||||
"github.com/ultravioletrs/agent/agent/api"
|
||||
agentgrpc "github.com/ultravioletrs/agent/agent/api/grpc"
|
||||
httpapi "github.com/ultravioletrs/agent/agent/api/http"
|
||||
"github.com/ultravioletrs/agent/agent/tracing"
|
||||
"github.com/ultravioletrs/agent/internal"
|
||||
"github.com/ultravioletrs/agent/internal/env"
|
||||
jaegerclient "github.com/ultravioletrs/agent/internal/jaeger"
|
||||
"github.com/ultravioletrs/agent/internal/server"
|
||||
grpcserver "github.com/ultravioletrs/agent/internal/server/grpc"
|
||||
httpserver "github.com/ultravioletrs/agent/internal/server/http"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/reflection"
|
||||
)
|
||||
|
||||
const (
|
||||
svcName = "agent"
|
||||
envPrefixHTTP = "AGENT_HTTP_"
|
||||
envPrefixGRPC = "AGENT_GRPC_"
|
||||
defSvcHTTPPort = "9031"
|
||||
defSvcGRPCPort = "7002"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
LogLevel string `env:"AGENT_LOG_LEVEL" envDefault:"info"`
|
||||
JaegerURL string `env:"AGENT_JAEGER_URL" envDefault:"http://localhost:14268/api/traces"`
|
||||
InstanceID string `env:"AGENT_INSTANCE_ID" envDefault:""`
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
|
||||
var cfg config
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
log.Fatalf("failed to load %s configuration : %s", svcName, err)
|
||||
}
|
||||
|
||||
logger, err := mflog.New(os.Stdout, cfg.LogLevel)
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
if cfg.InstanceID == "" {
|
||||
cfg.InstanceID, err = uuid.New().ID()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to generate instanceID: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID)
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err))
|
||||
}
|
||||
defer func() {
|
||||
if err := tp.Shutdown(ctx); err != nil {
|
||||
logger.Error(fmt.Sprintf("Error shutting down tracer provider: %v", err))
|
||||
}
|
||||
}()
|
||||
tracer := tp.Tracer(svcName)
|
||||
|
||||
svc := newService(logger, tracer)
|
||||
|
||||
var httpServerConfig = server.Config{Port: defSvcHTTPPort}
|
||||
if err := env.Parse(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
|
||||
logger.Fatal(fmt.Sprintf("failed to load %s gRPC server configuration : %s", svcName, err))
|
||||
}
|
||||
hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(svc, cfg.InstanceID), logger)
|
||||
|
||||
var grpcServerConfig = server.Config{Port: defSvcGRPCPort}
|
||||
if err := env.Parse(&grpcServerConfig, env.Options{Prefix: envPrefixGRPC}); err != nil {
|
||||
log.Fatalf("failed to load %s gRPC server configuration : %s", svcName, err.Error())
|
||||
}
|
||||
registerAgentServiceServer := func(srv *grpc.Server) {
|
||||
reflection.Register(srv)
|
||||
agent.RegisterAgentServiceServer(srv, agentgrpc.NewServer(svc))
|
||||
}
|
||||
gs := grpcserver.New(ctx, cancel, svcName, grpcServerConfig, registerAgentServiceServer, logger)
|
||||
|
||||
g.Go(func() error {
|
||||
return hs.Start()
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
return gs.Start()
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
return server.StopHandler(ctx, cancel, logger, svcName, hs, gs)
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
logger.Error(fmt.Sprintf("%s service terminated: %s", svcName, err))
|
||||
}
|
||||
}
|
||||
|
||||
func newService(logger mflog.Logger, tracer trace.Tracer) agent.Service {
|
||||
svc := agent.New()
|
||||
|
||||
svc = api.LoggingMiddleware(svc, logger)
|
||||
counter, latency := internal.MakeMetrics(svcName, "api")
|
||||
svc = api.MetricsMiddleware(svc, counter, latency)
|
||||
svc = tracing.New(svc, tracer)
|
||||
|
||||
return svc
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/mainflux/mainflux/logger"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/ultravioletrs/agent/cli"
|
||||
"github.com/ultravioletrs/agent/internal/env"
|
||||
"github.com/ultravioletrs/agent/pkg/clients/grpc"
|
||||
"github.com/ultravioletrs/agent/pkg/sdk"
|
||||
)
|
||||
|
||||
const (
|
||||
svcName = "cli"
|
||||
envPrefixAgentGRPC = "AGENT_GRPC_"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
LogLevel string `env:"AGENT_LOG_LEVEL" envDefault:"info"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var cfg config
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
log.Fatalf("failed to load %s configuration : %s", svcName, err)
|
||||
}
|
||||
|
||||
logger, err := logger.New(os.Stdout, cfg.LogLevel)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating logger: %s", err)
|
||||
}
|
||||
|
||||
agentGRPCConfig := grpc.Config{}
|
||||
if err := env.Parse(&agentGRPCConfig, env.Options{Prefix: envPrefixAgentGRPC}); err != nil {
|
||||
logger.Fatal(fmt.Sprintf("failed to load %s gRPC client configuration : %s", svcName, err))
|
||||
}
|
||||
|
||||
agentGRPCClient, agentClient, err := grpc.NewClient(agentGRPCConfig)
|
||||
if err != nil {
|
||||
logger.Fatal(err.Error())
|
||||
}
|
||||
defer agentGRPCClient.Close()
|
||||
|
||||
sdk := sdk.NewAgentSDK(logger, agentClient)
|
||||
|
||||
cli.SetSDK(sdk)
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "agent-cli [command]",
|
||||
Short: "CLI application for Computation Service API",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("CLI application for Computation Service API\n\n")
|
||||
fmt.Printf("Usage:\n %s [command]\n\n", cmd.CommandPath())
|
||||
fmt.Printf("Available Commands:\n")
|
||||
|
||||
// Filter out "completion" command
|
||||
availableCommands := make([]*cobra.Command, 0)
|
||||
for _, subCmd := range cmd.Commands() {
|
||||
if subCmd.Name() != "completion" {
|
||||
availableCommands = append(availableCommands, subCmd)
|
||||
}
|
||||
}
|
||||
|
||||
for _, subCmd := range availableCommands {
|
||||
fmt.Printf(" %-15s%s\n", subCmd.Name(), subCmd.Short)
|
||||
}
|
||||
|
||||
fmt.Printf("\nFlags:\n")
|
||||
cmd.Flags().VisitAll(func(flag *pflag.Flag) {
|
||||
fmt.Printf(" -%s, --%s %s\n", flag.Shorthand, flag.Name, flag.Usage)
|
||||
})
|
||||
fmt.Printf("\nUse \"%s [command] --help\" for more information about a command.\n", cmd.CommandPath())
|
||||
},
|
||||
}
|
||||
|
||||
// Root Commands
|
||||
rootCmd.AddCommand(cli.NewAlgorithmsCmd(sdk))
|
||||
rootCmd.AddCommand(cli.NewDatasetsCmd(sdk))
|
||||
rootCmd.AddCommand(cli.NewResultsCmd(sdk))
|
||||
rootCmd.AddCommand(cli.NewRunCmd(sdk))
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
logger.Error(fmt.Sprintf("Command execution failed: %s", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
module github.com/ultravioletrs/agent
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/caarlos0/env/v7 v7.1.0
|
||||
github.com/go-kit/kit v0.12.0
|
||||
github.com/go-zoo/bone v1.3.0
|
||||
github.com/golang/protobuf v1.5.3
|
||||
github.com/mainflux/mainflux v0.0.0-20230726142711-2b78902e0170
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/spf13/cobra v1.7.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
|
||||
go.opentelemetry.io/contrib/propagators/jaeger v1.17.0
|
||||
go.opentelemetry.io/otel v1.16.0
|
||||
go.opentelemetry.io/otel/exporters/jaeger v1.16.0
|
||||
go.opentelemetry.io/otel/sdk v1.16.0
|
||||
go.opentelemetry.io/otel/trace v1.16.0
|
||||
golang.org/x/sync v0.3.0
|
||||
google.golang.org/grpc v1.57.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.3 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.6.0 // indirect
|
||||
github.com/go-logr/logr v1.2.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/gofrs/uuid v4.4.0+incompatible // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/prometheus/client_model v0.4.0 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.16.0 // indirect
|
||||
golang.org/x/net v0.12.0 // indirect
|
||||
golang.org/x/sys v0.10.0 // indirect
|
||||
golang.org/x/text v0.11.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e // indirect
|
||||
)
|
||||
@@ -0,0 +1,101 @@
|
||||
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
|
||||
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
|
||||
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/caarlos0/env/v7 v7.1.0 h1:9lzTF5amyQeWHZzuZeKlCb5FWSUxpG1js43mhbY8ozg=
|
||||
github.com/caarlos0/env/v7 v7.1.0/go.mod h1:LPPWniDUq4JaO6Q41vtlyikhMknqymCLBw0eX4dcH1E=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8=
|
||||
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
|
||||
github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4=
|
||||
github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs=
|
||||
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
|
||||
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
|
||||
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-zoo/bone v1.3.0 h1:PY6sHq37FnQhj+4ZyqFIzJQHvrrGx0GEc3vTZZC/OsI=
|
||||
github.com/go-zoo/bone v1.3.0/go.mod h1:HI3Lhb7G3UQcAwEhOJ2WyNcsFtQX1WYHa0Hl4OBbhW8=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/mainflux/mainflux v0.0.0-20230726142711-2b78902e0170 h1:lMvLFbtNJdUufGGYaC/MrNH+B1uEYs4LWstTjphWnbc=
|
||||
github.com/mainflux/mainflux v0.0.0-20230726142711-2b78902e0170/go.mod h1:FoeJ13mrfikrsFDW6bOb3C44D5gZ5m9Jt249G1sLKq0=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
|
||||
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
|
||||
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
|
||||
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
|
||||
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
|
||||
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
|
||||
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
|
||||
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
|
||||
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
|
||||
github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8=
|
||||
go.opentelemetry.io/contrib/propagators/jaeger v1.17.0 h1:Zbpbmwav32Ea5jSotpmkWEl3a6Xvd4tw/3xxGO1i05Y=
|
||||
go.opentelemetry.io/contrib/propagators/jaeger v1.17.0/go.mod h1:tcTUAlmO8nuInPDSBVfG+CP6Mzjy5+gNV4mPxMbL0IA=
|
||||
go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
|
||||
go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
|
||||
go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA=
|
||||
go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM=
|
||||
go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
|
||||
go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
|
||||
go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE=
|
||||
go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
|
||||
go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
|
||||
go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
|
||||
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
|
||||
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e h1:S83+ibolgyZ0bqz7KEsUOPErxcv4VzlszxY+31OfB/E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
|
||||
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
|
||||
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
// Package env contains utility tools for handling
|
||||
// environment-based service configuration.
|
||||
package env
|
||||
Vendored
+36
@@ -0,0 +1,36 @@
|
||||
package env
|
||||
|
||||
import "github.com/caarlos0/env/v7"
|
||||
|
||||
type Options struct {
|
||||
// Environment keys and values that will be accessible for the service
|
||||
Environment map[string]string
|
||||
|
||||
// TagName specifies another tagname to use rather than the default env
|
||||
TagName string
|
||||
|
||||
// RequiredIfNoDef automatically sets all env as required if they do not declare 'envDefault'
|
||||
RequiredIfNoDef bool
|
||||
|
||||
// OnSet allows to run a function when a value is set
|
||||
OnSet env.OnSetFn
|
||||
|
||||
// Prefix define a prefix for each key
|
||||
Prefix string
|
||||
}
|
||||
|
||||
func Parse(v interface{}, opts ...Options) error {
|
||||
var actOpts []env.Options
|
||||
|
||||
for _, opt := range opts {
|
||||
actOpts = append(actOpts, env.Options{
|
||||
Environment: opt.Environment,
|
||||
TagName: opt.TagName,
|
||||
RequiredIfNoDef: opt.RequiredIfNoDef,
|
||||
OnSet: opt.OnSet,
|
||||
Prefix: opt.Prefix,
|
||||
})
|
||||
}
|
||||
|
||||
return env.Parse(v, actOpts...)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package jaeger contains the domain concept definitions needed to support
|
||||
// Mainflux Jaeger functionality.
|
||||
package jaeger
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) Mainflux
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package jaeger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
jaegerp "go.opentelemetry.io/contrib/propagators/jaeger"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/exporters/jaeger"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
tracesdk "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
|
||||
)
|
||||
|
||||
var (
|
||||
errNoURL = errors.New("URL is empty")
|
||||
errNoSvcName = errors.New("Service Name is empty")
|
||||
)
|
||||
|
||||
// NewProvider initializes Jaeger TraceProvider.
|
||||
func NewProvider(ctx context.Context, svcName, url, instanceID string) (*tracesdk.TracerProvider, error) {
|
||||
if url == "" {
|
||||
return nil, errNoURL
|
||||
}
|
||||
|
||||
if svcName == "" {
|
||||
return nil, errNoSvcName
|
||||
}
|
||||
|
||||
exporter, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
attributes := []attribute.KeyValue{
|
||||
semconv.ServiceNameKey.String(svcName),
|
||||
attribute.String("InstanceID", instanceID),
|
||||
}
|
||||
|
||||
hostAttr, err := resource.New(ctx, resource.WithHost(), resource.WithOSDescription(), resource.WithContainer())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attributes = append(attributes, hostAttr.Attributes()...)
|
||||
|
||||
tp := tracesdk.NewTracerProvider(
|
||||
tracesdk.WithSampler(tracesdk.AlwaysSample()),
|
||||
tracesdk.WithBatcher(exporter),
|
||||
tracesdk.WithResource(resource.NewWithAttributes(
|
||||
semconv.SchemaURL,
|
||||
attributes...,
|
||||
)),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetTextMapPropagator(jaegerp.Jaeger{})
|
||||
|
||||
return tp, nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
kitprometheus "github.com/go-kit/kit/metrics/prometheus"
|
||||
stdprometheus "github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// MakeMetrics returns an instance of metrics.
|
||||
func MakeMetrics(namespace, subsystem string) (*kitprometheus.Counter, *kitprometheus.Summary) {
|
||||
counter := kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "request_count",
|
||||
Help: "Number of requests received.",
|
||||
}, []string{"method"})
|
||||
latency := kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
Name: "request_latency_microseconds",
|
||||
Help: "Total duration of requests in microseconds.",
|
||||
}, []string{"method"})
|
||||
|
||||
return counter, latency
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package server contains the HTTP, gRPC and CoAP server implementation.
|
||||
package server
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package grpc contains the gRPC server implementation.
|
||||
package grpc
|
||||
@@ -0,0 +1,102 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/mainflux/mainflux/logger"
|
||||
"github.com/ultravioletrs/agent/internal/server"
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
)
|
||||
|
||||
const (
|
||||
stopWaitTime = 5 * time.Second
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
server.BaseServer
|
||||
server *grpc.Server
|
||||
registerService serviceRegister
|
||||
}
|
||||
|
||||
type serviceRegister func(srv *grpc.Server)
|
||||
|
||||
var _ server.Server = (*Server)(nil)
|
||||
|
||||
func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger logger.Logger) server.Server {
|
||||
var listenFullAddress = fmt.Sprintf("%s:%s", config.Host, config.Port)
|
||||
|
||||
return &Server{
|
||||
BaseServer: server.BaseServer{
|
||||
Ctx: ctx,
|
||||
Cancel: cancel,
|
||||
Name: name,
|
||||
Address: listenFullAddress,
|
||||
Config: config,
|
||||
Logger: logger,
|
||||
},
|
||||
registerService: registerService,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Start() error {
|
||||
var errCh = make(chan error)
|
||||
|
||||
listener, err := net.Listen("tcp", s.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
|
||||
}
|
||||
|
||||
switch {
|
||||
case s.Config.CertFile != "" || s.Config.KeyFile != "":
|
||||
creds, err := credentials.NewServerTLSFromFile(s.Config.CertFile, s.Config.KeyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load auth certificates: %w", err)
|
||||
}
|
||||
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile))
|
||||
s.server = grpc.NewServer(
|
||||
grpc.Creds(creds),
|
||||
grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
|
||||
)
|
||||
default:
|
||||
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address))
|
||||
s.server = grpc.NewServer(
|
||||
grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
|
||||
)
|
||||
}
|
||||
|
||||
s.registerService(s.server)
|
||||
|
||||
go func() {
|
||||
errCh <- s.server.Serve(listener)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-s.Ctx.Done():
|
||||
return s.Stop()
|
||||
case err := <-errCh:
|
||||
s.Cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Stop() error {
|
||||
defer s.Cancel()
|
||||
var c = make(chan bool)
|
||||
go func() {
|
||||
defer close(c)
|
||||
s.server.GracefulStop()
|
||||
}()
|
||||
select {
|
||||
case <-c:
|
||||
case <-time.After(stopWaitTime):
|
||||
}
|
||||
s.Logger.Info(fmt.Sprintf("%s gRPC service shutdown at %s", s.Name, s.Address))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package http contains the HTTP server implementation.
|
||||
package http
|
||||
@@ -0,0 +1,76 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/mainflux/mainflux/logger"
|
||||
"github.com/ultravioletrs/agent/internal/server"
|
||||
)
|
||||
|
||||
const (
|
||||
stopWaitTime = 5 * time.Second
|
||||
httpProtocol = "http"
|
||||
httpsProtocol = "https"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
server.BaseServer
|
||||
server *http.Server
|
||||
}
|
||||
|
||||
var _ server.Server = (*Server)(nil)
|
||||
|
||||
func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, handler http.Handler, logger logger.Logger) server.Server {
|
||||
var listenFullAddress = fmt.Sprintf("%s:%s", config.Host, config.Port)
|
||||
var httpServer = &http.Server{Addr: listenFullAddress, Handler: handler}
|
||||
return &Server{
|
||||
BaseServer: server.BaseServer{
|
||||
Ctx: ctx,
|
||||
Cancel: cancel,
|
||||
Name: name,
|
||||
Address: listenFullAddress,
|
||||
Config: config,
|
||||
Logger: logger,
|
||||
},
|
||||
server: httpServer,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Start() error {
|
||||
var errCh = make(chan error)
|
||||
s.Protocol = httpProtocol
|
||||
switch {
|
||||
case s.Config.CertFile != "" || s.Config.KeyFile != "":
|
||||
s.Protocol = httpsProtocol
|
||||
s.Logger.Info(fmt.Sprintf("%s service %s server listening at %s with TLS cert %s and key %s", s.Name, s.Protocol, s.Address, s.Config.CertFile, s.Config.KeyFile))
|
||||
go func() {
|
||||
errCh <- s.server.ListenAndServeTLS(s.Config.CertFile, s.Config.KeyFile)
|
||||
}()
|
||||
default:
|
||||
s.Logger.Info(fmt.Sprintf("%s service %s server listening at %s without TLS", s.Name, s.Protocol, s.Address))
|
||||
go func() {
|
||||
errCh <- s.server.ListenAndServe()
|
||||
}()
|
||||
}
|
||||
select {
|
||||
case <-s.Ctx.Done():
|
||||
return s.Stop()
|
||||
case err := <-errCh:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Stop() error {
|
||||
defer s.Cancel()
|
||||
ctxShutdown, cancelShutdown := context.WithTimeout(context.Background(), stopWaitTime)
|
||||
defer cancelShutdown()
|
||||
if err := s.server.Shutdown(ctxShutdown); err != nil {
|
||||
s.Logger.Error(fmt.Sprintf("%s service %s server error occurred during shutdown at %s: %s", s.Name, s.Protocol, s.Address, err))
|
||||
return fmt.Errorf("%s service %s server error occurred during shutdown at %s: %w", s.Name, s.Protocol, s.Address, err)
|
||||
}
|
||||
s.Logger.Info(fmt.Sprintf("%s %s service shutdown of http at %s", s.Name, s.Protocol, s.Address))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/mainflux/mainflux/logger"
|
||||
)
|
||||
|
||||
type Server interface {
|
||||
Start() error
|
||||
Stop() error
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Host string `env:"HOST" envDefault:""`
|
||||
Port string `env:"PORT" envDefault:""`
|
||||
CertFile string `env:"SERVER_CERT" envDefault:""`
|
||||
KeyFile string `env:"SERVER_KEY" envDefault:""`
|
||||
}
|
||||
|
||||
type BaseServer struct {
|
||||
Ctx context.Context
|
||||
Cancel context.CancelFunc
|
||||
Name string
|
||||
Address string
|
||||
Config Config
|
||||
Logger logger.Logger
|
||||
Protocol string
|
||||
}
|
||||
|
||||
func stopAllServers(servers ...Server) error {
|
||||
var errs []error
|
||||
for _, server := range servers {
|
||||
if err := server.Stop(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("encountered errors while stopping servers: %v", errs)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func StopHandler(ctx context.Context, cancel context.CancelFunc, logger logger.Logger, svcName string, servers ...Server) error {
|
||||
var err error
|
||||
var c = make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGINT, syscall.SIGABRT)
|
||||
select {
|
||||
case sig := <-c:
|
||||
defer cancel()
|
||||
err = stopAllServers(servers...)
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("%s service error during shutdown: %v", svcName, err))
|
||||
}
|
||||
logger.Info(fmt.Sprintf("%s service shutdown by signal: %s", svcName, sig))
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package clients contains the domain concept definitions needed to support
|
||||
// Agent Client functionality.
|
||||
package clients
|
||||
@@ -0,0 +1,16 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
agentapi "github.com/ultravioletrs/agent/agent/api/grpc"
|
||||
)
|
||||
|
||||
// NewClient creates new agent gRPC client instance.
|
||||
func NewClient(cfg Config) (Client, agent.AgentServiceClient, error) {
|
||||
client, err := newClient(cfg)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client, agentapi.NewClient(client.Connection(), cfg.Timeout), nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/mainflux/mainflux/pkg/errors"
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||
gogrpc "google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
var (
|
||||
errGrpcConnect = errors.New("failed to connect to grpc server")
|
||||
errGrpcClose = errors.New("failed to close grpc connection")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ClientTLS bool `env:"CLIENT_TLS" envDefault:"false"`
|
||||
CACerts string `env:"CA_CERTS" envDefault:""`
|
||||
URL string `env:"URL" envDefault:"localhost:7020"`
|
||||
Timeout time.Duration `env:"TIMEOUT" envDefault:"60s"`
|
||||
}
|
||||
|
||||
type Client interface {
|
||||
// Close closes gRPC connection.
|
||||
Close() error
|
||||
|
||||
// Secure is used for pretty printing TLS info.
|
||||
Secure() string
|
||||
|
||||
// Connection returns the gRPC connection.
|
||||
Connection() *gogrpc.ClientConn
|
||||
}
|
||||
|
||||
type client struct {
|
||||
*gogrpc.ClientConn
|
||||
cfg Config
|
||||
secure bool
|
||||
}
|
||||
|
||||
var _ Client = (*client)(nil)
|
||||
|
||||
func newClient(cfg Config) (Client, error) {
|
||||
conn, secure, err := connect(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &client{
|
||||
ClientConn: conn,
|
||||
cfg: cfg,
|
||||
secure: secure,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *client) Close() error {
|
||||
if err := c.ClientConn.Close(); err != nil {
|
||||
return errors.Wrap(errGrpcClose, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) Secure() string {
|
||||
if c.secure {
|
||||
return "with TLS"
|
||||
}
|
||||
return "without TLS"
|
||||
}
|
||||
|
||||
func (c *client) Connection() *gogrpc.ClientConn {
|
||||
return c.ClientConn
|
||||
}
|
||||
|
||||
// connect creates new gRPC client and connect to gRPC server.
|
||||
func connect(cfg Config) (*gogrpc.ClientConn, bool, error) {
|
||||
var opts []gogrpc.DialOption
|
||||
secure := false
|
||||
tc := insecure.NewCredentials()
|
||||
|
||||
if cfg.ClientTLS && cfg.CACerts != "" {
|
||||
var err error
|
||||
tc, err = credentials.NewClientTLSFromFile(cfg.CACerts, "")
|
||||
if err != nil {
|
||||
return nil, secure, err
|
||||
}
|
||||
secure = true
|
||||
}
|
||||
|
||||
opts = append(opts, gogrpc.WithTransportCredentials(tc), gogrpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()))
|
||||
|
||||
conn, err := gogrpc.Dial(cfg.URL, opts...)
|
||||
if err != nil {
|
||||
return nil, secure, errors.Wrap(errGrpcConnect, err)
|
||||
}
|
||||
|
||||
return conn, secure, nil
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
// Package grpc contains the domain concept definitions needed to support
|
||||
// Agent Client grpc functionality.
|
||||
package grpc
|
||||
@@ -0,0 +1,107 @@
|
||||
package sdk
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/mainflux/mainflux/logger"
|
||||
"github.com/ultravioletrs/agent/agent"
|
||||
)
|
||||
|
||||
type SDK interface {
|
||||
Run(computation Computation) (string, error)
|
||||
UploadAlgorithm(algorithm []byte) (string, error)
|
||||
UploadDataset(dataset []byte) (string, error)
|
||||
Result() ([]byte, error)
|
||||
}
|
||||
|
||||
type agentSDK struct {
|
||||
client agent.AgentServiceClient
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
type Computation struct {
|
||||
ID string `json:"id,omitempty" db:"id"`
|
||||
Name string `json:"name,omitempty" db:"name"`
|
||||
Description string `json:"description,omitempty" db:"description"`
|
||||
Status string `json:"status,omitempty" db:"status"`
|
||||
Owner string `json:"owner,omitempty" db:"owner"`
|
||||
StartTime time.Time `json:"start_time,omitempty" db:"start_time"`
|
||||
EndTime time.Time `json:"end_time,omitempty" db:"end_time"`
|
||||
Datasets []string `json:"datasets,omitempty" db:"datasets"`
|
||||
Algorithms []string `json:"algorithms,omitempty" db:"algorithms"`
|
||||
DatasetProviders []string `json:"dataset_providers,omitempty" db:"dataset_providers"`
|
||||
AlgorithmProviders []string `json:"algorithm_providers,omitempty" db:"algorithm_providers"`
|
||||
ResultConsumers []string `json:"result_consumers,omitempty" db:"result_consumers"`
|
||||
Ttl int `json:"ttl,omitempty" db:"ttl"`
|
||||
Metadata Metadata `json:"metadata,omitempty" db:"metadata"`
|
||||
}
|
||||
|
||||
type Metadata map[string]interface{}
|
||||
|
||||
func NewAgentSDK(log logger.Logger, agentClient agent.AgentServiceClient) *agentSDK {
|
||||
return &agentSDK{
|
||||
client: agentClient,
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (sdk *agentSDK) Run(computation Computation) (string, error) {
|
||||
computationBytes, err := json.Marshal(computation)
|
||||
if err != nil {
|
||||
sdk.logger.Error("Failed to marshal computation")
|
||||
return "", err
|
||||
}
|
||||
|
||||
request := &agent.RunRequest{
|
||||
Computation: computationBytes,
|
||||
}
|
||||
response, err := sdk.client.Run(context.Background(), request)
|
||||
if err != nil {
|
||||
sdk.logger.Error("Failed to call Run RPC")
|
||||
return "", err
|
||||
}
|
||||
|
||||
return response.Computation, nil
|
||||
}
|
||||
|
||||
func (sdk *agentSDK) UploadAlgorithm(algorithm []byte) (string, error) {
|
||||
request := &agent.AlgoRequest{
|
||||
Algorithm: algorithm,
|
||||
}
|
||||
|
||||
response, err := sdk.client.Algo(context.Background(), request)
|
||||
if err != nil {
|
||||
sdk.logger.Error("Failed to call Algo RPC")
|
||||
return "", err
|
||||
}
|
||||
|
||||
return response.AlgorithmID, nil
|
||||
}
|
||||
|
||||
func (sdk *agentSDK) UploadDataset(dataset []byte) (string, error) {
|
||||
request := &agent.DataRequest{
|
||||
Dataset: dataset,
|
||||
}
|
||||
|
||||
response, err := sdk.client.Data(context.Background(), request)
|
||||
if err != nil {
|
||||
sdk.logger.Error("Failed to call Data RPC")
|
||||
return "", err
|
||||
}
|
||||
|
||||
return response.DatasetID, nil
|
||||
}
|
||||
|
||||
func (sdk *agentSDK) Result() ([]byte, error) {
|
||||
request := &agent.ResultRequest{}
|
||||
|
||||
response, err := sdk.client.Result(context.Background(), request)
|
||||
if err != nil {
|
||||
sdk.logger.Error("Failed to call Result RPC")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return response.File, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
|
||||
func StartUnixSocketServer(socketPath string) (net.Listener, error) {
|
||||
// Remove any existing socket file
|
||||
_ = os.Remove(socketPath)
|
||||
|
||||
// Create a Unix domain socket listener
|
||||
listener, err := net.Listen("unix", socketPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating socket listener: %v", err)
|
||||
}
|
||||
|
||||
fmt.Println("Unix domain socket server is listening on", socketPath)
|
||||
|
||||
return listener, nil
|
||||
}
|
||||
|
||||
func AcceptConnection(listener net.Listener, dataChannel chan []byte, errorChannel chan error) {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
errorChannel <- fmt.Errorf("error accepting connection:: %v", err)
|
||||
}
|
||||
|
||||
handleConnection(conn, dataChannel, errorChannel)
|
||||
}
|
||||
|
||||
func handleConnection(conn net.Conn, dataChannel chan []byte, errorChannel chan error) {
|
||||
defer conn.Close()
|
||||
|
||||
// Create a dynamic buffer to store incoming data
|
||||
var buffer []byte
|
||||
tmp := make([]byte, 1024)
|
||||
|
||||
for {
|
||||
// Read data into the temporary buffer
|
||||
n, err := conn.Read(tmp)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
errorChannel <- err
|
||||
}
|
||||
buffer = append(buffer, tmp[:n]...)
|
||||
}
|
||||
|
||||
dataChannel <- buffer
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=Cocos AI agent
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
StandardOutput=file:/var/log/cocos/agent.stdout
|
||||
StandardError=file:/var/log/cocos/agent.stderr
|
||||
|
||||
Environment=NETWORK_INTERFACE=enp0s3
|
||||
Environment=AGENT_GRPC_HOST=10.0.2.15
|
||||
Environment=AGENT_GRPC_PORT=7002
|
||||
Environment=AGENT_LOG_LEVEL=info
|
||||
|
||||
ExecStartPre=ip link set dev $NETWORK_INTERFACE up
|
||||
ExecStartPre=dhclient $NETWORK_INTERFACE
|
||||
ExecStartPre=mkdir -p /var/log/cocos
|
||||
|
||||
ExecStart=/cocos/agent
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,70 @@
|
||||
# Manual tests
|
||||
|
||||
## CLI
|
||||
|
||||
Throughout the tests, we assume that our current working directory is the root of the `agent` repository, both on the host machine and in the VM.
|
||||
|
||||
### Python requirements
|
||||
|
||||
Do this both on the host machine and in the VM.
|
||||
|
||||
```sh
|
||||
apt update
|
||||
apt install python3-pip
|
||||
pip3 install pandas sklearn scikit-learn
|
||||
```
|
||||
|
||||
### Agent-CLI interaction
|
||||
|
||||
In the VM, open a console and start `agent`:
|
||||
|
||||
```sh
|
||||
AGENT_LOG_LEVEL=info AGENT_GRPC_URL=10.0.2.15:7002 go run cmd/agent/main.go
|
||||
```
|
||||
|
||||
Open console on the host, and run
|
||||
|
||||
```sh
|
||||
export AGENT_GRPC_URL=localhost:7020
|
||||
|
||||
# Run the CLI program with algorithm input
|
||||
go run cmd/cli/main.go algo test/manual/algo/lin_reg.py
|
||||
# 2023/09/21 10:43:53 Uploading algorithm binary: test/manual/algo/lin_reg.py
|
||||
|
||||
# Run the CLI program with dataset input
|
||||
go run cmd/cli/main.go data test/manual/data/iris.csv
|
||||
# 2023/09/21 10:45:25 Uploading dataset CSV: test/manual/data/iris.csv
|
||||
|
||||
# Run the CLI program to fetch computation result
|
||||
go run cmd/cli/main.go result
|
||||
# 2023/09/21 10:45:39 Retrieving computation result file
|
||||
# 2023/09/21 10:45:40 Computation result retrieved and saved successfully!
|
||||
```
|
||||
|
||||
Now there is a `result.bin` file in the current working directory. The file holds the trained logistic regression model. To test the model, run
|
||||
|
||||
```sh
|
||||
python3 test/manual/algo/lin_reg_test.py test/manual/data/iris.csv result.bin
|
||||
```
|
||||
|
||||
You should get an output (truncated for the sake of brevity):
|
||||
|
||||
```sh
|
||||
Id SepalLengthCm SepalWidthCm PetalLengthCm PetalWidthCm Species
|
||||
0 1 5.1 3.5 1.4 0.2 Iris-setosa
|
||||
1 2 4.9 3.0 1.4 0.2 Iris-setosa
|
||||
2 3 4.7 3.2 1.3 0.2 Iris-setosa
|
||||
3 4 4.6 3.1 1.5 0.2 Iris-setosa
|
||||
4 5 5.0 3.6 1.4 0.2 Iris-setosa
|
||||
Precision, Recall, Confusion matrix, in training
|
||||
|
||||
precision recall f1-score support
|
||||
|
||||
Iris-setosa 1.000 1.000 1.000 21
|
||||
Iris-versicolor 0.923 0.889 0.906 27
|
||||
Iris-virginica 0.893 0.926 0.909 27
|
||||
|
||||
accuracy 0.933 75
|
||||
macro avg 0.939 0.938 0.938 75
|
||||
weighted avg 0.934 0.933 0.933 75
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
import sys, io
|
||||
import joblib
|
||||
import socket
|
||||
|
||||
import pandas as pd
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
dataset = sys.argv[1]
|
||||
iris = pd.read_csv(io.StringIO(dataset))
|
||||
|
||||
# Droping the Species since we only need the measurements
|
||||
X = iris.drop(['Species'], axis=1)
|
||||
|
||||
# converting into numpy array and assigning petal length and petal width
|
||||
X = X.to_numpy()[:, (3,4)]
|
||||
y = iris['Species']
|
||||
|
||||
# Splitting into train and test
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.5, random_state=42)
|
||||
|
||||
log_reg = LogisticRegression()
|
||||
log_reg.fit(X_train,y_train)
|
||||
|
||||
# Serialize the trained model to a byte buffer
|
||||
model_buffer = io.BytesIO()
|
||||
joblib.dump(log_reg, model_buffer)
|
||||
|
||||
# Get the serialized model as a bytes object
|
||||
model_bytes = model_buffer.getvalue()
|
||||
|
||||
# Define the path for the Unix domain socket
|
||||
socket_path = sys.argv[2]
|
||||
|
||||
# Create a Unix domain socket client
|
||||
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
|
||||
try:
|
||||
# Connect to the server
|
||||
client.connect(socket_path)
|
||||
|
||||
# Send the serialized model over the socket
|
||||
client.send(model_bytes)
|
||||
|
||||
finally:
|
||||
# Close the socket
|
||||
client.close()
|
||||
@@ -0,0 +1,51 @@
|
||||
import pandas as pd
|
||||
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn import metrics
|
||||
import joblib
|
||||
|
||||
import sys
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
|
||||
csv_file_path = sys.argv[1]
|
||||
model_filename = sys.argv[2]
|
||||
|
||||
# Load the CSV file into a Pandas DataFrame
|
||||
iris = pd.read_csv(csv_file_path)
|
||||
|
||||
log_reg = joblib.load(model_filename)
|
||||
|
||||
# Now you have the Iris dataset loaded into the iris_df DataFrame
|
||||
print(iris.head()) # Display the first few rows of the DataFrame
|
||||
|
||||
# Droping the Species since we only need the measurements
|
||||
X = iris.drop(['Species'], axis=1)
|
||||
|
||||
# converting into numpy array and assigning petal length and petal width
|
||||
X = X.to_numpy()[:, (3,4)]
|
||||
y = iris['Species']
|
||||
|
||||
# Splitting into train and test
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.5, random_state=42)
|
||||
|
||||
training_prediction = log_reg.predict(X_train)
|
||||
test_prediction = log_reg.predict(X_test)
|
||||
|
||||
print("Precision, Recall, Confusion matrix, in training\n")
|
||||
|
||||
# Precision Recall scores
|
||||
print(metrics.classification_report(y_train, training_prediction, digits=3))
|
||||
|
||||
# Confusion matrix
|
||||
print(metrics.confusion_matrix(y_train, training_prediction))
|
||||
|
||||
print("Precision, Recall, Confusion matrix, in testing\n")
|
||||
|
||||
# Precision Recall scores
|
||||
print(metrics.classification_report(y_test, test_prediction, digits=3))
|
||||
|
||||
# Confusion matrix
|
||||
print(metrics.confusion_matrix(y_test, test_prediction))
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
Id,SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm,Species
|
||||
1,5.1,3.5,1.4,0.2,Iris-setosa
|
||||
2,4.9,3.0,1.4,0.2,Iris-setosa
|
||||
3,4.7,3.2,1.3,0.2,Iris-setosa
|
||||
4,4.6,3.1,1.5,0.2,Iris-setosa
|
||||
5,5.0,3.6,1.4,0.2,Iris-setosa
|
||||
6,5.4,3.9,1.7,0.4,Iris-setosa
|
||||
7,4.6,3.4,1.4,0.3,Iris-setosa
|
||||
8,5.0,3.4,1.5,0.2,Iris-setosa
|
||||
9,4.4,2.9,1.4,0.2,Iris-setosa
|
||||
10,4.9,3.1,1.5,0.1,Iris-setosa
|
||||
11,5.4,3.7,1.5,0.2,Iris-setosa
|
||||
12,4.8,3.4,1.6,0.2,Iris-setosa
|
||||
13,4.8,3.0,1.4,0.1,Iris-setosa
|
||||
14,4.3,3.0,1.1,0.1,Iris-setosa
|
||||
15,5.8,4.0,1.2,0.2,Iris-setosa
|
||||
16,5.7,4.4,1.5,0.4,Iris-setosa
|
||||
17,5.4,3.9,1.3,0.4,Iris-setosa
|
||||
18,5.1,3.5,1.4,0.3,Iris-setosa
|
||||
19,5.7,3.8,1.7,0.3,Iris-setosa
|
||||
20,5.1,3.8,1.5,0.3,Iris-setosa
|
||||
21,5.4,3.4,1.7,0.2,Iris-setosa
|
||||
22,5.1,3.7,1.5,0.4,Iris-setosa
|
||||
23,4.6,3.6,1.0,0.2,Iris-setosa
|
||||
24,5.1,3.3,1.7,0.5,Iris-setosa
|
||||
25,4.8,3.4,1.9,0.2,Iris-setosa
|
||||
26,5.0,3.0,1.6,0.2,Iris-setosa
|
||||
27,5.0,3.4,1.6,0.4,Iris-setosa
|
||||
28,5.2,3.5,1.5,0.2,Iris-setosa
|
||||
29,5.2,3.4,1.4,0.2,Iris-setosa
|
||||
30,4.7,3.2,1.6,0.2,Iris-setosa
|
||||
31,4.8,3.1,1.6,0.2,Iris-setosa
|
||||
32,5.4,3.4,1.5,0.4,Iris-setosa
|
||||
33,5.2,4.1,1.5,0.1,Iris-setosa
|
||||
34,5.5,4.2,1.4,0.2,Iris-setosa
|
||||
35,4.9,3.1,1.5,0.1,Iris-setosa
|
||||
36,5.0,3.2,1.2,0.2,Iris-setosa
|
||||
37,5.5,3.5,1.3,0.2,Iris-setosa
|
||||
38,4.9,3.1,1.5,0.1,Iris-setosa
|
||||
39,4.4,3.0,1.3,0.2,Iris-setosa
|
||||
40,5.1,3.4,1.5,0.2,Iris-setosa
|
||||
41,5.0,3.5,1.3,0.3,Iris-setosa
|
||||
42,4.5,2.3,1.3,0.3,Iris-setosa
|
||||
43,4.4,3.2,1.3,0.2,Iris-setosa
|
||||
44,5.0,3.5,1.6,0.6,Iris-setosa
|
||||
45,5.1,3.8,1.9,0.4,Iris-setosa
|
||||
46,4.8,3.0,1.4,0.3,Iris-setosa
|
||||
47,5.1,3.8,1.6,0.2,Iris-setosa
|
||||
48,4.6,3.2,1.4,0.2,Iris-setosa
|
||||
49,5.3,3.7,1.5,0.2,Iris-setosa
|
||||
50,5.0,3.3,1.4,0.2,Iris-setosa
|
||||
51,7.0,3.2,4.7,1.4,Iris-versicolor
|
||||
52,6.4,3.2,4.5,1.5,Iris-versicolor
|
||||
53,6.9,3.1,4.9,1.5,Iris-versicolor
|
||||
54,5.5,2.3,4.0,1.3,Iris-versicolor
|
||||
55,6.5,2.8,4.6,1.5,Iris-versicolor
|
||||
56,5.7,2.8,4.5,1.3,Iris-versicolor
|
||||
57,6.3,3.3,4.7,1.6,Iris-versicolor
|
||||
58,4.9,2.4,3.3,1.0,Iris-versicolor
|
||||
59,6.6,2.9,4.6,1.3,Iris-versicolor
|
||||
60,5.2,2.7,3.9,1.4,Iris-versicolor
|
||||
61,5.0,2.0,3.5,1.0,Iris-versicolor
|
||||
62,5.9,3.0,4.2,1.5,Iris-versicolor
|
||||
63,6.0,2.2,4.0,1.0,Iris-versicolor
|
||||
64,6.1,2.9,4.7,1.4,Iris-versicolor
|
||||
65,5.6,2.9,3.6,1.3,Iris-versicolor
|
||||
66,6.7,3.1,4.4,1.4,Iris-versicolor
|
||||
67,5.6,3.0,4.5,1.5,Iris-versicolor
|
||||
68,5.8,2.7,4.1,1.0,Iris-versicolor
|
||||
69,6.2,2.2,4.5,1.5,Iris-versicolor
|
||||
70,5.6,2.5,3.9,1.1,Iris-versicolor
|
||||
71,5.9,3.2,4.8,1.8,Iris-versicolor
|
||||
72,6.1,2.8,4.0,1.3,Iris-versicolor
|
||||
73,6.3,2.5,4.9,1.5,Iris-versicolor
|
||||
74,6.1,2.8,4.7,1.2,Iris-versicolor
|
||||
75,6.4,2.9,4.3,1.3,Iris-versicolor
|
||||
76,6.6,3.0,4.4,1.4,Iris-versicolor
|
||||
77,6.8,2.8,4.8,1.4,Iris-versicolor
|
||||
78,6.7,3.0,5.0,1.7,Iris-versicolor
|
||||
79,6.0,2.9,4.5,1.5,Iris-versicolor
|
||||
80,5.7,2.6,3.5,1.0,Iris-versicolor
|
||||
81,5.5,2.4,3.8,1.1,Iris-versicolor
|
||||
82,5.5,2.4,3.7,1.0,Iris-versicolor
|
||||
83,5.8,2.7,3.9,1.2,Iris-versicolor
|
||||
84,6.0,2.7,5.1,1.6,Iris-versicolor
|
||||
85,5.4,3.0,4.5,1.5,Iris-versicolor
|
||||
86,6.0,3.4,4.5,1.6,Iris-versicolor
|
||||
87,6.7,3.1,4.7,1.5,Iris-versicolor
|
||||
88,6.3,2.3,4.4,1.3,Iris-versicolor
|
||||
89,5.6,3.0,4.1,1.3,Iris-versicolor
|
||||
90,5.5,2.5,4.0,1.3,Iris-versicolor
|
||||
91,5.5,2.6,4.4,1.2,Iris-versicolor
|
||||
92,6.1,3.0,4.6,1.4,Iris-versicolor
|
||||
93,5.8,2.6,4.0,1.2,Iris-versicolor
|
||||
94,5.0,2.3,3.3,1.0,Iris-versicolor
|
||||
95,5.6,2.7,4.2,1.3,Iris-versicolor
|
||||
96,5.7,3.0,4.2,1.2,Iris-versicolor
|
||||
97,5.7,2.9,4.2,1.3,Iris-versicolor
|
||||
98,6.2,2.9,4.3,1.3,Iris-versicolor
|
||||
99,5.1,2.5,3.0,1.1,Iris-versicolor
|
||||
100,5.7,2.8,4.1,1.3,Iris-versicolor
|
||||
101,6.3,3.3,6.0,2.5,Iris-virginica
|
||||
102,5.8,2.7,5.1,1.9,Iris-virginica
|
||||
103,7.1,3.0,5.9,2.1,Iris-virginica
|
||||
104,6.3,2.9,5.6,1.8,Iris-virginica
|
||||
105,6.5,3.0,5.8,2.2,Iris-virginica
|
||||
106,7.6,3.0,6.6,2.1,Iris-virginica
|
||||
107,4.9,2.5,4.5,1.7,Iris-virginica
|
||||
108,7.3,2.9,6.3,1.8,Iris-virginica
|
||||
109,6.7,2.5,5.8,1.8,Iris-virginica
|
||||
110,7.2,3.6,6.1,2.5,Iris-virginica
|
||||
111,6.5,3.2,5.1,2.0,Iris-virginica
|
||||
112,6.4,2.7,5.3,1.9,Iris-virginica
|
||||
113,6.8,3.0,5.5,2.1,Iris-virginica
|
||||
114,5.7,2.5,5.0,2.0,Iris-virginica
|
||||
115,5.8,2.8,5.1,2.4,Iris-virginica
|
||||
116,6.4,3.2,5.3,2.3,Iris-virginica
|
||||
117,6.5,3.0,5.5,1.8,Iris-virginica
|
||||
118,7.7,3.8,6.7,2.2,Iris-virginica
|
||||
119,7.7,2.6,6.9,2.3,Iris-virginica
|
||||
120,6.0,2.2,5.0,1.5,Iris-virginica
|
||||
121,6.9,3.2,5.7,2.3,Iris-virginica
|
||||
122,5.6,2.8,4.9,2.0,Iris-virginica
|
||||
123,7.7,2.8,6.7,2.0,Iris-virginica
|
||||
124,6.3,2.7,4.9,1.8,Iris-virginica
|
||||
125,6.7,3.3,5.7,2.1,Iris-virginica
|
||||
126,7.2,3.2,6.0,1.8,Iris-virginica
|
||||
127,6.2,2.8,4.8,1.8,Iris-virginica
|
||||
128,6.1,3.0,4.9,1.8,Iris-virginica
|
||||
129,6.4,2.8,5.6,2.1,Iris-virginica
|
||||
130,7.2,3.0,5.8,1.6,Iris-virginica
|
||||
131,7.4,2.8,6.1,1.9,Iris-virginica
|
||||
132,7.9,3.8,6.4,2.0,Iris-virginica
|
||||
133,6.4,2.8,5.6,2.2,Iris-virginica
|
||||
134,6.3,2.8,5.1,1.5,Iris-virginica
|
||||
135,6.1,2.6,5.6,1.4,Iris-virginica
|
||||
136,7.7,3.0,6.1,2.3,Iris-virginica
|
||||
137,6.3,3.4,5.6,2.4,Iris-virginica
|
||||
138,6.4,3.1,5.5,1.8,Iris-virginica
|
||||
139,6.0,3.0,4.8,1.8,Iris-virginica
|
||||
140,6.9,3.1,5.4,2.1,Iris-virginica
|
||||
141,6.7,3.1,5.6,2.4,Iris-virginica
|
||||
142,6.9,3.1,5.1,2.3,Iris-virginica
|
||||
143,5.8,2.7,5.1,1.9,Iris-virginica
|
||||
144,6.8,3.2,5.9,2.3,Iris-virginica
|
||||
145,6.7,3.3,5.7,2.5,Iris-virginica
|
||||
146,6.7,3.0,5.2,2.3,Iris-virginica
|
||||
147,6.3,2.5,5.0,1.9,Iris-virginica
|
||||
148,6.5,3.0,5.2,2.0,Iris-virginica
|
||||
149,6.2,3.4,5.4,2.3,Iris-virginica
|
||||
150,5.9,3.0,5.1,1.8,Iris-virginica
|
||||
|
+20
@@ -0,0 +1,20 @@
|
||||
Copyright (C) 2013 Blake Mizerany
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
+2388
File diff suppressed because it is too large
Load Diff
+316
@@ -0,0 +1,316 @@
|
||||
// Package quantile computes approximate quantiles over an unbounded data
|
||||
// stream within low memory and CPU bounds.
|
||||
//
|
||||
// A small amount of accuracy is traded to achieve the above properties.
|
||||
//
|
||||
// Multiple streams can be merged before calling Query to generate a single set
|
||||
// of results. This is meaningful when the streams represent the same type of
|
||||
// data. See Merge and Samples.
|
||||
//
|
||||
// For more detailed information about the algorithm used, see:
|
||||
//
|
||||
// Effective Computation of Biased Quantiles over Data Streams
|
||||
//
|
||||
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
|
||||
package quantile
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Sample holds an observed value and meta information for compression. JSON
|
||||
// tags have been added for convenience.
|
||||
type Sample struct {
|
||||
Value float64 `json:",string"`
|
||||
Width float64 `json:",string"`
|
||||
Delta float64 `json:",string"`
|
||||
}
|
||||
|
||||
// Samples represents a slice of samples. It implements sort.Interface.
|
||||
type Samples []Sample
|
||||
|
||||
func (a Samples) Len() int { return len(a) }
|
||||
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
|
||||
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
|
||||
type invariant func(s *stream, r float64) float64
|
||||
|
||||
// NewLowBiased returns an initialized Stream for low-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the lower ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewLowBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * r
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewHighBiased returns an initialized Stream for high-biased quantiles
|
||||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
||||
// error guarantees can still be given even for the higher ranks of the data
|
||||
// distribution.
|
||||
//
|
||||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
||||
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
||||
// properties.
|
||||
func NewHighBiased(epsilon float64) *Stream {
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
return 2 * epsilon * (s.n - r)
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
// NewTargeted returns an initialized Stream concerned with a particular set of
|
||||
// quantile values that are supplied a priori. Knowing these a priori reduces
|
||||
// space and computation time. The targets map maps the desired quantiles to
|
||||
// their absolute errors, i.e. the true quantile of a value returned by a query
|
||||
// is guaranteed to be within (Quantile±Epsilon).
|
||||
//
|
||||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
|
||||
func NewTargeted(targetMap map[float64]float64) *Stream {
|
||||
// Convert map to slice to avoid slow iterations on a map.
|
||||
// ƒ is called on the hot path, so converting the map to a slice
|
||||
// beforehand results in significant CPU savings.
|
||||
targets := targetMapToSlice(targetMap)
|
||||
|
||||
ƒ := func(s *stream, r float64) float64 {
|
||||
var m = math.MaxFloat64
|
||||
var f float64
|
||||
for _, t := range targets {
|
||||
if t.quantile*s.n <= r {
|
||||
f = (2 * t.epsilon * r) / t.quantile
|
||||
} else {
|
||||
f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
|
||||
}
|
||||
if f < m {
|
||||
m = f
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
return newStream(ƒ)
|
||||
}
|
||||
|
||||
type target struct {
|
||||
quantile float64
|
||||
epsilon float64
|
||||
}
|
||||
|
||||
func targetMapToSlice(targetMap map[float64]float64) []target {
|
||||
targets := make([]target, 0, len(targetMap))
|
||||
|
||||
for quantile, epsilon := range targetMap {
|
||||
t := target{
|
||||
quantile: quantile,
|
||||
epsilon: epsilon,
|
||||
}
|
||||
targets = append(targets, t)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
|
||||
// design. Take care when using across multiple goroutines.
|
||||
type Stream struct {
|
||||
*stream
|
||||
b Samples
|
||||
sorted bool
|
||||
}
|
||||
|
||||
func newStream(ƒ invariant) *Stream {
|
||||
x := &stream{ƒ: ƒ}
|
||||
return &Stream{x, make(Samples, 0, 500), true}
|
||||
}
|
||||
|
||||
// Insert inserts v into the stream.
|
||||
func (s *Stream) Insert(v float64) {
|
||||
s.insert(Sample{Value: v, Width: 1})
|
||||
}
|
||||
|
||||
func (s *Stream) insert(sample Sample) {
|
||||
s.b = append(s.b, sample)
|
||||
s.sorted = false
|
||||
if len(s.b) == cap(s.b) {
|
||||
s.flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Query returns the computed qth percentiles value. If s was created with
|
||||
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
|
||||
// will return an unspecified result.
|
||||
func (s *Stream) Query(q float64) float64 {
|
||||
if !s.flushed() {
|
||||
// Fast path when there hasn't been enough data for a flush;
|
||||
// this also yields better accuracy for small sets of data.
|
||||
l := len(s.b)
|
||||
if l == 0 {
|
||||
return 0
|
||||
}
|
||||
i := int(math.Ceil(float64(l) * q))
|
||||
if i > 0 {
|
||||
i -= 1
|
||||
}
|
||||
s.maybeSort()
|
||||
return s.b[i].Value
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.query(q)
|
||||
}
|
||||
|
||||
// Merge merges samples into the underlying streams samples. This is handy when
|
||||
// merging multiple streams from separate threads, database shards, etc.
|
||||
//
|
||||
// ATTENTION: This method is broken and does not yield correct results. The
|
||||
// underlying algorithm is not capable of merging streams correctly.
|
||||
func (s *Stream) Merge(samples Samples) {
|
||||
sort.Sort(samples)
|
||||
s.stream.merge(samples)
|
||||
}
|
||||
|
||||
// Reset reinitializes and clears the list reusing the samples buffer memory.
|
||||
func (s *Stream) Reset() {
|
||||
s.stream.reset()
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
// Samples returns stream samples held by s.
|
||||
func (s *Stream) Samples() Samples {
|
||||
if !s.flushed() {
|
||||
return s.b
|
||||
}
|
||||
s.flush()
|
||||
return s.stream.samples()
|
||||
}
|
||||
|
||||
// Count returns the total number of samples observed in the stream
|
||||
// since initialization.
|
||||
func (s *Stream) Count() int {
|
||||
return len(s.b) + s.stream.count()
|
||||
}
|
||||
|
||||
func (s *Stream) flush() {
|
||||
s.maybeSort()
|
||||
s.stream.merge(s.b)
|
||||
s.b = s.b[:0]
|
||||
}
|
||||
|
||||
func (s *Stream) maybeSort() {
|
||||
if !s.sorted {
|
||||
s.sorted = true
|
||||
sort.Sort(s.b)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stream) flushed() bool {
|
||||
return len(s.stream.l) > 0
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
n float64
|
||||
l []Sample
|
||||
ƒ invariant
|
||||
}
|
||||
|
||||
func (s *stream) reset() {
|
||||
s.l = s.l[:0]
|
||||
s.n = 0
|
||||
}
|
||||
|
||||
func (s *stream) insert(v float64) {
|
||||
s.merge(Samples{{v, 1, 0}})
|
||||
}
|
||||
|
||||
func (s *stream) merge(samples Samples) {
|
||||
// TODO(beorn7): This tries to merge not only individual samples, but
|
||||
// whole summaries. The paper doesn't mention merging summaries at
|
||||
// all. Unittests show that the merging is inaccurate. Find out how to
|
||||
// do merges properly.
|
||||
var r float64
|
||||
i := 0
|
||||
for _, sample := range samples {
|
||||
for ; i < len(s.l); i++ {
|
||||
c := s.l[i]
|
||||
if c.Value > sample.Value {
|
||||
// Insert at position i.
|
||||
s.l = append(s.l, Sample{})
|
||||
copy(s.l[i+1:], s.l[i:])
|
||||
s.l[i] = Sample{
|
||||
sample.Value,
|
||||
sample.Width,
|
||||
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
|
||||
// TODO(beorn7): How to calculate delta correctly?
|
||||
}
|
||||
i++
|
||||
goto inserted
|
||||
}
|
||||
r += c.Width
|
||||
}
|
||||
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
|
||||
i++
|
||||
inserted:
|
||||
s.n += sample.Width
|
||||
r += sample.Width
|
||||
}
|
||||
s.compress()
|
||||
}
|
||||
|
||||
func (s *stream) count() int {
|
||||
return int(s.n)
|
||||
}
|
||||
|
||||
func (s *stream) query(q float64) float64 {
|
||||
t := math.Ceil(q * s.n)
|
||||
t += math.Ceil(s.ƒ(s, t) / 2)
|
||||
p := s.l[0]
|
||||
var r float64
|
||||
for _, c := range s.l[1:] {
|
||||
r += p.Width
|
||||
if r+c.Width+c.Delta > t {
|
||||
return p.Value
|
||||
}
|
||||
p = c
|
||||
}
|
||||
return p.Value
|
||||
}
|
||||
|
||||
func (s *stream) compress() {
|
||||
if len(s.l) < 2 {
|
||||
return
|
||||
}
|
||||
x := s.l[len(s.l)-1]
|
||||
xi := len(s.l) - 1
|
||||
r := s.n - 1 - x.Width
|
||||
|
||||
for i := len(s.l) - 2; i >= 0; i-- {
|
||||
c := s.l[i]
|
||||
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
|
||||
x.Width += c.Width
|
||||
s.l[xi] = x
|
||||
// Remove element at i.
|
||||
copy(s.l[i:], s.l[i+1:])
|
||||
s.l = s.l[:len(s.l)-1]
|
||||
xi -= 1
|
||||
} else {
|
||||
x = c
|
||||
xi = i
|
||||
}
|
||||
r -= c.Width
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stream) samples() Samples {
|
||||
samples := make(Samples, len(s.l))
|
||||
copy(samples, s.l)
|
||||
return samples
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
coverage.txt
|
||||
bin
|
||||
card.png
|
||||
dist
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
linters:
|
||||
enable:
|
||||
- thelper
|
||||
- gofumpt
|
||||
- tparallel
|
||||
- unconvert
|
||||
- unparam
|
||||
- wastedassign
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
includes:
|
||||
- from_url:
|
||||
url: https://raw.githubusercontent.com/caarlos0/.goreleaserfiles/main/lib.yml
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2022 Carlos Alexandro Becker
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
SOURCE_FILES?=./...
|
||||
TEST_PATTERN?=.
|
||||
|
||||
export GO111MODULE := on
|
||||
|
||||
setup:
|
||||
go mod tidy
|
||||
.PHONY: setup
|
||||
|
||||
build:
|
||||
go build
|
||||
.PHONY: build
|
||||
|
||||
test:
|
||||
go test -v -failfast -race -coverpkg=./... -covermode=atomic -coverprofile=coverage.txt $(SOURCE_FILES) -run $(TEST_PATTERN) -timeout=2m
|
||||
.PHONY: test
|
||||
|
||||
cover: test
|
||||
go tool cover -html=coverage.txt
|
||||
.PHONY: cover
|
||||
|
||||
fmt:
|
||||
gofumpt -w -l .
|
||||
.PHONY: fmt
|
||||
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
.PHONY: lint
|
||||
|
||||
ci: build test
|
||||
.PHONY: ci
|
||||
|
||||
card:
|
||||
wget -O card.png -c "https://og.caarlos0.dev/**env**: parse envs to structs.png?theme=light&md=1&fontSize=100px&images=https://github.com/caarlos0.png"
|
||||
.PHONY: card
|
||||
|
||||
.DEFAULT_GOAL := ci
|
||||
+551
@@ -0,0 +1,551 @@
|
||||
# env
|
||||
|
||||
[](https://github.com/caarlos0/env/actions?workflow=build)
|
||||
[](https://codecov.io/gh/caarlos0/env)
|
||||
[](https://pkg.go.dev/github.com/caarlos0/env/v7)
|
||||
|
||||
A simple and zero-dependencies library to parse environment variables into structs.
|
||||
|
||||
## Example
|
||||
|
||||
Get the module with:
|
||||
|
||||
```sh
|
||||
go get github.com/caarlos0/env/v7
|
||||
```
|
||||
|
||||
The usage looks like this:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Home string `env:"HOME"`
|
||||
Port int `env:"PORT" envDefault:"3000"`
|
||||
Password string `env:"PASSWORD,unset"`
|
||||
IsProduction bool `env:"PRODUCTION"`
|
||||
Hosts []string `env:"HOSTS" envSeparator:":"`
|
||||
Duration time.Duration `env:"DURATION"`
|
||||
TempFolder string `env:"TEMP_FOLDER" envDefault:"${HOME}/tmp" envExpand:"true"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := config{}
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
fmt.Printf("%+v\n", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
You can run it like this:
|
||||
|
||||
```sh
|
||||
$ PRODUCTION=true HOSTS="host1:host2:host3" DURATION=1s go run main.go
|
||||
{Home:/your/home Port:3000 IsProduction:true Hosts:[host1 host2 host3] Duration:1s}
|
||||
```
|
||||
|
||||
## Caveats
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
> **This is important!**
|
||||
|
||||
- _Unexported fields_ are **ignored**
|
||||
|
||||
|
||||
## Supported types and defaults
|
||||
|
||||
Out of the box all built-in types are supported, plus a few others that
|
||||
are commonly used.
|
||||
|
||||
Complete list:
|
||||
|
||||
- `string`
|
||||
- `bool`
|
||||
- `int`
|
||||
- `int8`
|
||||
- `int16`
|
||||
- `int32`
|
||||
- `int64`
|
||||
- `uint`
|
||||
- `uint8`
|
||||
- `uint16`
|
||||
- `uint32`
|
||||
- `uint64`
|
||||
- `float32`
|
||||
- `float64`
|
||||
- `time.Duration`
|
||||
- `encoding.TextUnmarshaler`
|
||||
- `url.URL`
|
||||
|
||||
Pointers, slices and slices of pointers, and maps of those types are also
|
||||
supported.
|
||||
|
||||
You can also use/define a [custom parser func](#custom-parser-funcs) for any
|
||||
other type you want.
|
||||
|
||||
You can also use custom keys and values in your maps, as long as you provide a
|
||||
parser function for them.
|
||||
|
||||
If you set the `envDefault` tag for something, this value will be used in the
|
||||
case of absence of it in the environment.
|
||||
|
||||
By default, slice types will split the environment value on `,`; you can change
|
||||
this behavior by setting the `envSeparator` tag.
|
||||
|
||||
If you set the `envExpand` tag, environment variables (either in `${var}` or
|
||||
`$var` format) in the string will be replaced according with the actual value
|
||||
of the variable.
|
||||
|
||||
## Custom Parser Funcs
|
||||
|
||||
If you have a type that is not supported out of the box by the lib, you are able
|
||||
to use (or define) and pass custom parsers (and their associated `reflect.Type`)
|
||||
to the `env.ParseWithFuncs()` function.
|
||||
|
||||
In addition to accepting a struct pointer (same as `Parse()`), this function
|
||||
also accepts a `map[reflect.Type]env.ParserFunc`.
|
||||
|
||||
If you add a custom parser for, say `Foo`, it will also be used to parse
|
||||
`*Foo` and `[]Foo` types.
|
||||
|
||||
Check the examples in the [go doc](http://pkg.go.dev/github.com/caarlos0/env/v7)
|
||||
for more info.
|
||||
|
||||
### A note about `TextUnmarshaler` and `time.Time`
|
||||
|
||||
Env supports by default anything that implements the `TextUnmarshaler` interface.
|
||||
That includes things like `time.Time` for example.
|
||||
The upside is that depending on the format you need, you don't need to change anything.
|
||||
The downside is that if you do need time in another format, you'll need to create your own type.
|
||||
|
||||
Its fairly straightforward:
|
||||
|
||||
```go
|
||||
type MyTime time.Time
|
||||
|
||||
func (t *MyTime) UnmarshalText(text []byte) error {
|
||||
tt, err := time.Parse("2006-01-02", string(text))
|
||||
*t = MyTime(tt)
|
||||
return err
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
SomeTime MyTime `env:"SOME_TIME"`
|
||||
}
|
||||
```
|
||||
|
||||
And then you can parse `Config` with `env.Parse`.
|
||||
|
||||
## Required fields
|
||||
|
||||
The `env` tag option `required` (e.g., `env:"tagKey,required"`) can be added to ensure that some environment variable is set.
|
||||
In the example above, an error is returned if the `config` struct is changed to:
|
||||
|
||||
```go
|
||||
type config struct {
|
||||
SecretKey string `env:"SECRET_KEY,required"`
|
||||
}
|
||||
```
|
||||
|
||||
## Not Empty fields
|
||||
|
||||
While `required` demands the environment variable to be set, it doesn't check its value.
|
||||
If you want to make sure the environment is set and not empty, you need to use the `notEmpty` tag option instead (`env:"SOME_ENV,notEmpty"`).
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
type config struct {
|
||||
SecretKey string `env:"SECRET_KEY,notEmpty"`
|
||||
}
|
||||
```
|
||||
|
||||
## Unset environment variable after reading it
|
||||
|
||||
The `env` tag option `unset` (e.g., `env:"tagKey,unset"`) can be added
|
||||
to ensure that some environment variable is unset after reading it.
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
type config struct {
|
||||
SecretKey string `env:"SECRET_KEY,unset"`
|
||||
}
|
||||
```
|
||||
|
||||
## From file
|
||||
|
||||
The `env` tag option `file` (e.g., `env:"tagKey,file"`) can be added
|
||||
to in order to indicate that the value of the variable shall be loaded from a file. The path of that file is given
|
||||
by the environment variable associated with it
|
||||
Example below
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Secret string `env:"SECRET,file"`
|
||||
Password string `env:"PASSWORD,file" envDefault:"/tmp/password"`
|
||||
Certificate string `env:"CERTIFICATE,file" envDefault:"${CERTIFICATE_FILE}" envExpand:"true"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := config{}
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
fmt.Printf("%+v\n", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
```sh
|
||||
$ echo qwerty > /tmp/secret
|
||||
$ echo dvorak > /tmp/password
|
||||
$ echo coleman > /tmp/certificate
|
||||
|
||||
$ SECRET=/tmp/secret \
|
||||
CERTIFICATE_FILE=/tmp/certificate \
|
||||
go run main.go
|
||||
{Secret:qwerty Password:dvorak Certificate:coleman}
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
### Use field names as environment variables by default
|
||||
|
||||
If you don't want to set the `env` tag on every field, you can use the
|
||||
`UseFieldNameByDefault` option.
|
||||
|
||||
It will use the field name as environment variable name.
|
||||
|
||||
Here's an example:
|
||||
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Username string // will use $USERNAME
|
||||
Password string // will use $PASSWORD
|
||||
UserFullName string // will use $USER_FULL_NAME
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := &Config{}
|
||||
opts := &env.Options{UseFieldNameByDefault: true}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
### Environment
|
||||
|
||||
By setting the `Options.Environment` map you can tell `Parse` to add those `keys` and `values`
|
||||
as env vars before parsing is done. These envs are stored in the map and never actually set by `os.Setenv`.
|
||||
This option effectively makes `env` ignore the OS environment variables: only the ones provided in the option are used.
|
||||
|
||||
This can make your testing scenarios a bit more clean and easy to handle.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Password string `env:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := &Config{}
|
||||
opts := &env.Options{Environment: map[string]string{
|
||||
"PASSWORD": "MY_PASSWORD",
|
||||
}}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
### Changing default tag name
|
||||
|
||||
You can change what tag name to use for setting the env vars by setting the `Options.TagName`
|
||||
variable.
|
||||
|
||||
For example
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Password string `json:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := &Config{}
|
||||
opts := &env.Options{TagName: "json"}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
### Prefixes
|
||||
|
||||
You can prefix sub-structs env tags, as well as a whole `env.Parse` call.
|
||||
|
||||
Here's an example flexing it a bit:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Home string `env:"HOME"`
|
||||
}
|
||||
|
||||
type ComplexConfig struct {
|
||||
Foo Config `envPrefix:"FOO_"`
|
||||
Clean Config
|
||||
Bar Config `envPrefix:"BAR_"`
|
||||
Blah string `env:"BLAH"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := ComplexConfig{}
|
||||
if err := Parse(&cfg, Options{
|
||||
Prefix: "T_",
|
||||
Environment: map[string]string{
|
||||
"T_FOO_HOME": "/foo",
|
||||
"T_BAR_HOME": "/bar",
|
||||
"T_BLAH": "blahhh",
|
||||
"T_HOME": "/clean",
|
||||
},
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
### On set hooks
|
||||
|
||||
You might want to listen to value sets and, for example, log something or do some other kind of logic.
|
||||
You can do this by passing a `OnSet` option:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Username string `env:"USERNAME" envDefault:"admin"`
|
||||
Password string `env:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := &Config{}
|
||||
opts := &env.Options{
|
||||
OnSet: func(tag string, value interface{}, isDefault bool) {
|
||||
fmt.Printf("Set %s to %v (default? %v)\n", tag, value, isDefault)
|
||||
},
|
||||
}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
## Making all fields to required
|
||||
|
||||
You can make all fields that don't have a default value be required by setting the `RequiredIfNoDef: true` in the `Options`.
|
||||
|
||||
For example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Username string `env:"USERNAME" envDefault:"admin"`
|
||||
Password string `env:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := &Config{}
|
||||
opts := &env.Options{RequiredIfNoDef: true}
|
||||
|
||||
// Load env vars.
|
||||
if err := env.Parse(cfg, opts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Print the loaded data.
|
||||
fmt.Printf("%+v\n", cfg)
|
||||
}
|
||||
```
|
||||
|
||||
## Defaults from code
|
||||
|
||||
You may define default value also in code, by initialising the config data before it's filled by `env.Parse`.
|
||||
Default values defined as struct tags will overwrite existing values during Parse.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Username string `env:"USERNAME" envDefault:"admin"`
|
||||
Password string `env:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var cfg = Config{
|
||||
Username: "test",
|
||||
Password: "123456",
|
||||
}
|
||||
|
||||
if err := env.Parse(&cfg); err != nil {
|
||||
fmt.Println("failed:", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%+v", cfg) // {Username:admin Password:123456}
|
||||
}
|
||||
```
|
||||
|
||||
## Error handling
|
||||
|
||||
You can handle the errors the library throws like so:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/caarlos0/env/v7"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Username string `env:"USERNAME" envDefault:"admin"`
|
||||
Password string `env:"PASSWORD"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var cfg Config
|
||||
err := env.Parse(&cfg)
|
||||
if e, ok := err.(*env.AggregateError); ok {
|
||||
for _, er := range e.Errors {
|
||||
switch v := er.(type) {
|
||||
case env.ParseError:
|
||||
// handle it
|
||||
case env.NotStructPtrError:
|
||||
// handle it
|
||||
case env.NoParserError:
|
||||
// handle it
|
||||
case env.NoSupportedTagOptionError:
|
||||
// handle it
|
||||
default:
|
||||
fmt.Printf("Unknown error type %v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("%+v", cfg) // {Username:admin Password:123456}
|
||||
}
|
||||
```
|
||||
|
||||
> **Info**
|
||||
>
|
||||
> If you want to check if an specific error is in the chain, you can also use
|
||||
> `errors.Is()`.
|
||||
|
||||
## Stargazers over time
|
||||
|
||||
[](https://starchart.cc/caarlos0/env)
|
||||
+540
@@ -0,0 +1,540 @@
|
||||
package env
|
||||
|
||||
import (
|
||||
"encoding"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// nolint: gochecknoglobals
|
||||
var (
|
||||
defaultBuiltInParsers = map[reflect.Kind]ParserFunc{
|
||||
reflect.Bool: func(v string) (interface{}, error) {
|
||||
return strconv.ParseBool(v)
|
||||
},
|
||||
reflect.String: func(v string) (interface{}, error) {
|
||||
return v, nil
|
||||
},
|
||||
reflect.Int: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseInt(v, 10, 32)
|
||||
return int(i), err
|
||||
},
|
||||
reflect.Int16: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseInt(v, 10, 16)
|
||||
return int16(i), err
|
||||
},
|
||||
reflect.Int32: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseInt(v, 10, 32)
|
||||
return int32(i), err
|
||||
},
|
||||
reflect.Int64: func(v string) (interface{}, error) {
|
||||
return strconv.ParseInt(v, 10, 64)
|
||||
},
|
||||
reflect.Int8: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseInt(v, 10, 8)
|
||||
return int8(i), err
|
||||
},
|
||||
reflect.Uint: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseUint(v, 10, 32)
|
||||
return uint(i), err
|
||||
},
|
||||
reflect.Uint16: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseUint(v, 10, 16)
|
||||
return uint16(i), err
|
||||
},
|
||||
reflect.Uint32: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseUint(v, 10, 32)
|
||||
return uint32(i), err
|
||||
},
|
||||
reflect.Uint64: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseUint(v, 10, 64)
|
||||
return i, err
|
||||
},
|
||||
reflect.Uint8: func(v string) (interface{}, error) {
|
||||
i, err := strconv.ParseUint(v, 10, 8)
|
||||
return uint8(i), err
|
||||
},
|
||||
reflect.Float64: func(v string) (interface{}, error) {
|
||||
return strconv.ParseFloat(v, 64)
|
||||
},
|
||||
reflect.Float32: func(v string) (interface{}, error) {
|
||||
f, err := strconv.ParseFloat(v, 32)
|
||||
return float32(f), err
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func defaultTypeParsers() map[reflect.Type]ParserFunc {
|
||||
return map[reflect.Type]ParserFunc{
|
||||
reflect.TypeOf(url.URL{}): func(v string) (interface{}, error) {
|
||||
u, err := url.Parse(v)
|
||||
if err != nil {
|
||||
return nil, newParseValueError("unable to parse URL", err)
|
||||
}
|
||||
return *u, nil
|
||||
},
|
||||
reflect.TypeOf(time.Nanosecond): func(v string) (interface{}, error) {
|
||||
s, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
return nil, newParseValueError("unable to parse duration", err)
|
||||
}
|
||||
return s, err
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ParserFunc defines the signature of a function that can be used within `CustomParsers`.
|
||||
type ParserFunc func(v string) (interface{}, error)
|
||||
|
||||
// OnSetFn is a hook that can be run when a value is set.
|
||||
type OnSetFn func(tag string, value interface{}, isDefault bool)
|
||||
|
||||
// Options for the parser.
|
||||
type Options struct {
|
||||
// Environment keys and values that will be accessible for the service.
|
||||
Environment map[string]string
|
||||
|
||||
// TagName specifies another tagname to use rather than the default env.
|
||||
TagName string
|
||||
|
||||
// RequiredIfNoDef automatically sets all env as required if they do not
|
||||
// declare 'envDefault'.
|
||||
RequiredIfNoDef bool
|
||||
|
||||
// OnSet allows to run a function when a value is set.
|
||||
OnSet OnSetFn
|
||||
|
||||
// Prefix define a prefix for each key.
|
||||
Prefix string
|
||||
|
||||
// UseFieldNameByDefault defines whether or not env should use the field
|
||||
// name by default if the `env` key is missing.
|
||||
UseFieldNameByDefault bool
|
||||
|
||||
// Sets to true if we have already configured once.
|
||||
configured bool
|
||||
}
|
||||
|
||||
// configure will do the basic configurations and defaults.
|
||||
func configure(opts []Options) []Options {
|
||||
// If we have already configured the first item
|
||||
// of options will have been configured set to true.
|
||||
if len(opts) > 0 && opts[0].configured {
|
||||
return opts
|
||||
}
|
||||
|
||||
// Created options with defaults.
|
||||
opt := Options{
|
||||
TagName: "env",
|
||||
Environment: toMap(os.Environ()),
|
||||
configured: true,
|
||||
}
|
||||
|
||||
// Loop over all opts structs and set
|
||||
// to opt if value is not default/empty.
|
||||
for _, item := range opts {
|
||||
if item.Environment != nil {
|
||||
opt.Environment = item.Environment
|
||||
}
|
||||
if item.TagName != "" {
|
||||
opt.TagName = item.TagName
|
||||
}
|
||||
if item.OnSet != nil {
|
||||
opt.OnSet = item.OnSet
|
||||
}
|
||||
if item.Prefix != "" {
|
||||
opt.Prefix = item.Prefix
|
||||
}
|
||||
opt.UseFieldNameByDefault = item.UseFieldNameByDefault
|
||||
opt.RequiredIfNoDef = item.RequiredIfNoDef
|
||||
}
|
||||
|
||||
return []Options{opt}
|
||||
}
|
||||
|
||||
func getOnSetFn(opts []Options) OnSetFn {
|
||||
return opts[0].OnSet
|
||||
}
|
||||
|
||||
// getTagName returns the tag name.
|
||||
func getTagName(opts []Options) string {
|
||||
return opts[0].TagName
|
||||
}
|
||||
|
||||
// getEnvironment returns the environment map.
|
||||
func getEnvironment(opts []Options) map[string]string {
|
||||
return opts[0].Environment
|
||||
}
|
||||
|
||||
// Parse parses a struct containing `env` tags and loads its values from
|
||||
// environment variables.
|
||||
func Parse(v interface{}, opts ...Options) error {
|
||||
return ParseWithFuncs(v, map[reflect.Type]ParserFunc{}, opts...)
|
||||
}
|
||||
|
||||
// ParseWithFuncs is the same as `Parse` except it also allows the user to pass
|
||||
// in custom parsers.
|
||||
func ParseWithFuncs(v interface{}, funcMap map[reflect.Type]ParserFunc, opts ...Options) error {
|
||||
opts = configure(opts)
|
||||
|
||||
ptrRef := reflect.ValueOf(v)
|
||||
if ptrRef.Kind() != reflect.Ptr {
|
||||
return newAggregateError(NotStructPtrError{})
|
||||
}
|
||||
ref := ptrRef.Elem()
|
||||
if ref.Kind() != reflect.Struct {
|
||||
return newAggregateError(NotStructPtrError{})
|
||||
}
|
||||
parsers := defaultTypeParsers()
|
||||
for k, v := range funcMap {
|
||||
parsers[k] = v
|
||||
}
|
||||
|
||||
return doParse(ref, parsers, opts)
|
||||
}
|
||||
|
||||
func doParse(ref reflect.Value, funcMap map[reflect.Type]ParserFunc, opts []Options) error {
|
||||
refType := ref.Type()
|
||||
|
||||
var agrErr AggregateError
|
||||
|
||||
for i := 0; i < refType.NumField(); i++ {
|
||||
refField := ref.Field(i)
|
||||
refTypeField := refType.Field(i)
|
||||
|
||||
if err := doParseField(refField, refTypeField, funcMap, opts); err != nil {
|
||||
if val, ok := err.(AggregateError); ok {
|
||||
agrErr.Errors = append(agrErr.Errors, val.Errors...)
|
||||
} else {
|
||||
agrErr.Errors = append(agrErr.Errors, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(agrErr.Errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return agrErr
|
||||
}
|
||||
|
||||
func doParseField(refField reflect.Value, refTypeField reflect.StructField, funcMap map[reflect.Type]ParserFunc, opts []Options) error {
|
||||
if !refField.CanSet() {
|
||||
return nil
|
||||
}
|
||||
if reflect.Ptr == refField.Kind() && !refField.IsNil() {
|
||||
return ParseWithFuncs(refField.Interface(), funcMap, optsWithPrefix(refTypeField, opts)...)
|
||||
}
|
||||
if reflect.Struct == refField.Kind() && refField.CanAddr() && refField.Type().Name() == "" {
|
||||
return ParseWithFuncs(refField.Addr().Interface(), funcMap, optsWithPrefix(refTypeField, opts)...)
|
||||
}
|
||||
value, err := get(refTypeField, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if value != "" {
|
||||
return set(refField, refTypeField, value, funcMap)
|
||||
}
|
||||
|
||||
if reflect.Struct == refField.Kind() {
|
||||
return doParse(refField, funcMap, optsWithPrefix(refTypeField, opts))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const underscore rune = '_'
|
||||
|
||||
func toEnvName(input string) string {
|
||||
var output []rune
|
||||
for i, c := range input {
|
||||
if i > 0 && output[i-1] != underscore && c != underscore && unicode.ToUpper(c) == c {
|
||||
output = append(output, underscore)
|
||||
}
|
||||
output = append(output, unicode.ToUpper(c))
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func get(field reflect.StructField, opts []Options) (val string, err error) {
|
||||
var exists bool
|
||||
var isDefault bool
|
||||
var loadFile bool
|
||||
var unset bool
|
||||
var notEmpty bool
|
||||
|
||||
required := opts[0].RequiredIfNoDef
|
||||
prefix := opts[0].Prefix
|
||||
ownKey, tags := parseKeyForOption(field.Tag.Get(getTagName(opts)))
|
||||
if ownKey == "" && opts[0].UseFieldNameByDefault {
|
||||
ownKey = toEnvName(field.Name)
|
||||
}
|
||||
key := prefix + ownKey
|
||||
for _, tag := range tags {
|
||||
switch tag {
|
||||
case "":
|
||||
continue
|
||||
case "file":
|
||||
loadFile = true
|
||||
case "required":
|
||||
required = true
|
||||
case "unset":
|
||||
unset = true
|
||||
case "notEmpty":
|
||||
notEmpty = true
|
||||
default:
|
||||
return "", newNoSupportedTagOptionError(tag)
|
||||
}
|
||||
}
|
||||
expand := strings.EqualFold(field.Tag.Get("envExpand"), "true")
|
||||
defaultValue, defExists := field.Tag.Lookup("envDefault")
|
||||
val, exists, isDefault = getOr(key, defaultValue, defExists, getEnvironment(opts))
|
||||
|
||||
if expand {
|
||||
val = os.ExpandEnv(val)
|
||||
}
|
||||
|
||||
if unset {
|
||||
defer os.Unsetenv(key)
|
||||
}
|
||||
|
||||
if required && !exists && len(ownKey) > 0 {
|
||||
return "", newEnvVarIsNotSet(key)
|
||||
}
|
||||
|
||||
if notEmpty && val == "" {
|
||||
return "", newEmptyEnvVarError(key)
|
||||
}
|
||||
|
||||
if loadFile && val != "" {
|
||||
filename := val
|
||||
val, err = getFromFile(filename)
|
||||
if err != nil {
|
||||
return "", newLoadFileContentError(filename, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
if onSetFn := getOnSetFn(opts); onSetFn != nil {
|
||||
onSetFn(key, val, isDefault)
|
||||
}
|
||||
return val, err
|
||||
}
|
||||
|
||||
// split the env tag's key into the expected key and desired option, if any.
|
||||
func parseKeyForOption(key string) (string, []string) {
|
||||
opts := strings.Split(key, ",")
|
||||
return opts[0], opts[1:]
|
||||
}
|
||||
|
||||
func getFromFile(filename string) (value string, err error) {
|
||||
b, err := os.ReadFile(filename)
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
func getOr(key, defaultValue string, defExists bool, envs map[string]string) (string, bool, bool) {
|
||||
value, exists := envs[key]
|
||||
switch {
|
||||
case (!exists || key == "") && defExists:
|
||||
return defaultValue, true, true
|
||||
case exists && value == "" && defExists:
|
||||
return defaultValue, true, true
|
||||
case !exists:
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
return value, true, false
|
||||
}
|
||||
|
||||
func set(field reflect.Value, sf reflect.StructField, value string, funcMap map[reflect.Type]ParserFunc) error {
|
||||
if tm := asTextUnmarshaler(field); tm != nil {
|
||||
if err := tm.UnmarshalText([]byte(value)); err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
typee := sf.Type
|
||||
fieldee := field
|
||||
if typee.Kind() == reflect.Ptr {
|
||||
typee = typee.Elem()
|
||||
fieldee = field.Elem()
|
||||
}
|
||||
|
||||
parserFunc, ok := funcMap[typee]
|
||||
if ok {
|
||||
val, err := parserFunc(value)
|
||||
if err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
|
||||
fieldee.Set(reflect.ValueOf(val))
|
||||
return nil
|
||||
}
|
||||
|
||||
parserFunc, ok = defaultBuiltInParsers[typee.Kind()]
|
||||
if ok {
|
||||
val, err := parserFunc(value)
|
||||
if err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
|
||||
fieldee.Set(reflect.ValueOf(val).Convert(typee))
|
||||
return nil
|
||||
}
|
||||
|
||||
switch field.Kind() {
|
||||
case reflect.Slice:
|
||||
return handleSlice(field, value, sf, funcMap)
|
||||
case reflect.Map:
|
||||
return handleMap(field, value, sf, funcMap)
|
||||
}
|
||||
|
||||
return newNoParserError(sf)
|
||||
}
|
||||
|
||||
func handleSlice(field reflect.Value, value string, sf reflect.StructField, funcMap map[reflect.Type]ParserFunc) error {
|
||||
separator := sf.Tag.Get("envSeparator")
|
||||
if separator == "" {
|
||||
separator = ","
|
||||
}
|
||||
parts := strings.Split(value, separator)
|
||||
|
||||
typee := sf.Type.Elem()
|
||||
if typee.Kind() == reflect.Ptr {
|
||||
typee = typee.Elem()
|
||||
}
|
||||
|
||||
if _, ok := reflect.New(typee).Interface().(encoding.TextUnmarshaler); ok {
|
||||
return parseTextUnmarshalers(field, parts, sf)
|
||||
}
|
||||
|
||||
parserFunc, ok := funcMap[typee]
|
||||
if !ok {
|
||||
parserFunc, ok = defaultBuiltInParsers[typee.Kind()]
|
||||
if !ok {
|
||||
return newNoParserError(sf)
|
||||
}
|
||||
}
|
||||
|
||||
result := reflect.MakeSlice(sf.Type, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
r, err := parserFunc(part)
|
||||
if err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
v := reflect.ValueOf(r).Convert(typee)
|
||||
if sf.Type.Elem().Kind() == reflect.Ptr {
|
||||
v = reflect.New(typee)
|
||||
v.Elem().Set(reflect.ValueOf(r).Convert(typee))
|
||||
}
|
||||
result = reflect.Append(result, v)
|
||||
}
|
||||
field.Set(result)
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleMap(field reflect.Value, value string, sf reflect.StructField, funcMap map[reflect.Type]ParserFunc) error {
|
||||
keyType := sf.Type.Key()
|
||||
keyParserFunc, ok := funcMap[keyType]
|
||||
if !ok {
|
||||
keyParserFunc, ok = defaultBuiltInParsers[keyType.Kind()]
|
||||
if !ok {
|
||||
return newNoParserError(sf)
|
||||
}
|
||||
}
|
||||
|
||||
elemType := sf.Type.Elem()
|
||||
elemParserFunc, ok := funcMap[elemType]
|
||||
if !ok {
|
||||
elemParserFunc, ok = defaultBuiltInParsers[elemType.Kind()]
|
||||
if !ok {
|
||||
return newNoParserError(sf)
|
||||
}
|
||||
}
|
||||
|
||||
separator := sf.Tag.Get("envSeparator")
|
||||
if separator == "" {
|
||||
separator = ","
|
||||
}
|
||||
|
||||
result := reflect.MakeMap(sf.Type)
|
||||
for _, part := range strings.Split(value, separator) {
|
||||
pairs := strings.Split(part, ":")
|
||||
if len(pairs) != 2 {
|
||||
return newParseError(sf, fmt.Errorf(`%q should be in "key:value" format`, part))
|
||||
}
|
||||
|
||||
key, err := keyParserFunc(pairs[0])
|
||||
if err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
|
||||
elem, err := elemParserFunc(pairs[1])
|
||||
if err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
|
||||
result.SetMapIndex(reflect.ValueOf(key).Convert(keyType), reflect.ValueOf(elem).Convert(elemType))
|
||||
}
|
||||
|
||||
field.Set(result)
|
||||
return nil
|
||||
}
|
||||
|
||||
func asTextUnmarshaler(field reflect.Value) encoding.TextUnmarshaler {
|
||||
if reflect.Ptr == field.Kind() {
|
||||
if field.IsNil() {
|
||||
field.Set(reflect.New(field.Type().Elem()))
|
||||
}
|
||||
} else if field.CanAddr() {
|
||||
field = field.Addr()
|
||||
}
|
||||
|
||||
tm, ok := field.Interface().(encoding.TextUnmarshaler)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return tm
|
||||
}
|
||||
|
||||
func parseTextUnmarshalers(field reflect.Value, data []string, sf reflect.StructField) error {
|
||||
s := len(data)
|
||||
elemType := field.Type().Elem()
|
||||
slice := reflect.MakeSlice(reflect.SliceOf(elemType), s, s)
|
||||
for i, v := range data {
|
||||
sv := slice.Index(i)
|
||||
kind := sv.Kind()
|
||||
if kind == reflect.Ptr {
|
||||
sv = reflect.New(elemType.Elem())
|
||||
} else {
|
||||
sv = sv.Addr()
|
||||
}
|
||||
tm := sv.Interface().(encoding.TextUnmarshaler)
|
||||
if err := tm.UnmarshalText([]byte(v)); err != nil {
|
||||
return newParseError(sf, err)
|
||||
}
|
||||
if kind == reflect.Ptr {
|
||||
slice.Index(i).Set(sv)
|
||||
}
|
||||
}
|
||||
|
||||
field.Set(slice)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func optsWithPrefix(field reflect.StructField, opts []Options) []Options {
|
||||
subOpts := make([]Options, len(opts))
|
||||
copy(subOpts, opts)
|
||||
if prefix := field.Tag.Get("envPrefix"); prefix != "" {
|
||||
subOpts[0].Prefix += prefix
|
||||
}
|
||||
return subOpts
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
|
||||
// +build darwin dragonfly freebsd linux netbsd openbsd solaris
|
||||
|
||||
package env
|
||||
|
||||
import "strings"
|
||||
|
||||
func toMap(env []string) map[string]string {
|
||||
r := map[string]string{}
|
||||
for _, e := range env {
|
||||
p := strings.SplitN(e, "=", 2)
|
||||
r[p[0]] = p[1]
|
||||
}
|
||||
return r
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package env
|
||||
|
||||
import "strings"
|
||||
|
||||
func toMap(env []string) map[string]string {
|
||||
r := map[string]string{}
|
||||
for _, e := range env {
|
||||
p := strings.SplitN(e, "=", 2)
|
||||
|
||||
// On Windows, environment variables can start with '='. If so, Split at next character.
|
||||
// See env_windows.go in the Go source: https://github.com/golang/go/blob/master/src/syscall/env_windows.go#L58
|
||||
prefixEqualSign := false
|
||||
if len(e) > 0 && e[0] == '=' {
|
||||
e = e[1:]
|
||||
prefixEqualSign = true
|
||||
}
|
||||
p = strings.SplitN(e, "=", 2)
|
||||
if prefixEqualSign {
|
||||
p[0] = "=" + p[0]
|
||||
}
|
||||
|
||||
r[p[0]] = p[1]
|
||||
}
|
||||
return r
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package env
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// An aggregated error wrapper to combine gathered errors. This allows either to display all errors or convert them individually
|
||||
// List of the available errors
|
||||
// ParseError
|
||||
// NotStructPtrError
|
||||
// NoParserError
|
||||
// NoSupportedTagOptionError
|
||||
// EnvVarIsNotSetError
|
||||
// EmptyEnvVarError
|
||||
// LoadFileContentError
|
||||
// ParseValueError
|
||||
type AggregateError struct {
|
||||
Errors []error
|
||||
}
|
||||
|
||||
func newAggregateError(initErr error) error {
|
||||
return AggregateError{
|
||||
[]error{
|
||||
initErr,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (e AggregateError) Error() string {
|
||||
var sb strings.Builder
|
||||
|
||||
sb.WriteString("env:")
|
||||
|
||||
for _, err := range e.Errors {
|
||||
sb.WriteString(fmt.Sprintf(" %v;", err.Error()))
|
||||
}
|
||||
|
||||
return strings.TrimRight(sb.String(), ";")
|
||||
}
|
||||
|
||||
// Is conforms with errors.Is.
|
||||
func (e AggregateError) Is(err error) bool {
|
||||
for _, ie := range e.Errors {
|
||||
if reflect.TypeOf(ie) == reflect.TypeOf(err) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The error occurs when it's impossible to convert the value for given type.
|
||||
type ParseError struct {
|
||||
Name string
|
||||
Type reflect.Type
|
||||
Err error
|
||||
}
|
||||
|
||||
func newParseError(sf reflect.StructField, err error) error {
|
||||
return ParseError{sf.Name, sf.Type, err}
|
||||
}
|
||||
|
||||
func (e ParseError) Error() string {
|
||||
return fmt.Sprintf(`parse error on field "%s" of type "%s": %v`, e.Name, e.Type, e.Err)
|
||||
}
|
||||
|
||||
// The error occurs when pass something that is not a pointer to a Struct to Parse
|
||||
type NotStructPtrError struct{}
|
||||
|
||||
func (e NotStructPtrError) Error() string {
|
||||
return "expected a pointer to a Struct"
|
||||
}
|
||||
|
||||
// This error occurs when there is no parser provided for given type
|
||||
// Supported types and defaults: https://github.com/caarlos0/env#supported-types-and-defaults
|
||||
// How to create a custom parser: https://github.com/caarlos0/env#custom-parser-funcs
|
||||
type NoParserError struct {
|
||||
Name string
|
||||
Type reflect.Type
|
||||
}
|
||||
|
||||
func newNoParserError(sf reflect.StructField) error {
|
||||
return NoParserError{sf.Name, sf.Type}
|
||||
}
|
||||
|
||||
func (e NoParserError) Error() string {
|
||||
return fmt.Sprintf(`no parser found for field "%s" of type "%s"`, e.Name, e.Type)
|
||||
}
|
||||
|
||||
// This error occurs when the given tag is not supported
|
||||
// In-built supported tags: "", "file", "required", "unset", "notEmpty", "envDefault", "envExpand", "envSeparator"
|
||||
// How to create a custom tag: https://github.com/caarlos0/env#changing-default-tag-name
|
||||
type NoSupportedTagOptionError struct {
|
||||
Tag string
|
||||
}
|
||||
|
||||
func newNoSupportedTagOptionError(tag string) error {
|
||||
return NoSupportedTagOptionError{tag}
|
||||
}
|
||||
|
||||
func (e NoSupportedTagOptionError) Error() string {
|
||||
return fmt.Sprintf("tag option %q not supported", e.Tag)
|
||||
}
|
||||
|
||||
// This error occurs when the required variable is not set
|
||||
// Read about required fields: https://github.com/caarlos0/env#required-fields
|
||||
type EnvVarIsNotSetError struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
func newEnvVarIsNotSet(key string) error {
|
||||
return EnvVarIsNotSetError{key}
|
||||
}
|
||||
|
||||
func (e EnvVarIsNotSetError) Error() string {
|
||||
return fmt.Sprintf(`required environment variable %q is not set`, e.Key)
|
||||
}
|
||||
|
||||
// This error occurs when the variable which must be not empty is existing but has an empty value
|
||||
// Read about not empty fields: https://github.com/caarlos0/env#not-empty-fields
|
||||
type EmptyEnvVarError struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
func newEmptyEnvVarError(key string) error {
|
||||
return EmptyEnvVarError{key}
|
||||
}
|
||||
|
||||
func (e EmptyEnvVarError) Error() string {
|
||||
return fmt.Sprintf("environment variable %q should not be empty", e.Key)
|
||||
}
|
||||
|
||||
// This error occurs when it's impossible to load the value from the file
|
||||
// Read about From file feature: https://github.com/caarlos0/env#from-file
|
||||
type LoadFileContentError struct {
|
||||
Filename string
|
||||
Key string
|
||||
Err error
|
||||
}
|
||||
|
||||
func newLoadFileContentError(filename, key string, err error) error {
|
||||
return LoadFileContentError{filename, key, err}
|
||||
}
|
||||
|
||||
func (e LoadFileContentError) Error() string {
|
||||
return fmt.Sprintf(`could not load content of file "%s" from variable %s: %v`, e.Filename, e.Key, e.Err)
|
||||
}
|
||||
|
||||
// This error occurs when it's impossible to convert value using given parser
|
||||
// Supported types and defaults: https://github.com/caarlos0/env#supported-types-and-defaults
|
||||
// How to create a custom parser: https://github.com/caarlos0/env#custom-parser-funcs
|
||||
type ParseValueError struct {
|
||||
Msg string
|
||||
Err error
|
||||
}
|
||||
|
||||
func newParseValueError(message string, err error) error {
|
||||
return ParseValueError{message, err}
|
||||
}
|
||||
|
||||
func (e ParseValueError) Error() string {
|
||||
return fmt.Sprintf("%s: %v", e.Msg, e.Err)
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
Copyright (c) 2016 Caleb Spare
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
# xxhash
|
||||
|
||||
[](https://pkg.go.dev/github.com/cespare/xxhash/v2)
|
||||
[](https://github.com/cespare/xxhash/actions/workflows/test.yml)
|
||||
|
||||
xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
|
||||
high-quality hashing algorithm that is much faster than anything in the Go
|
||||
standard library.
|
||||
|
||||
This package provides a straightforward API:
|
||||
|
||||
```
|
||||
func Sum64(b []byte) uint64
|
||||
func Sum64String(s string) uint64
|
||||
type Digest struct{ ... }
|
||||
func New() *Digest
|
||||
```
|
||||
|
||||
The `Digest` type implements hash.Hash64. Its key methods are:
|
||||
|
||||
```
|
||||
func (*Digest) Write([]byte) (int, error)
|
||||
func (*Digest) WriteString(string) (int, error)
|
||||
func (*Digest) Sum64() uint64
|
||||
```
|
||||
|
||||
The package is written with optimized pure Go and also contains even faster
|
||||
assembly implementations for amd64 and arm64. If desired, the `purego` build tag
|
||||
opts into using the Go code even on those architectures.
|
||||
|
||||
[xxHash]: http://cyan4973.github.io/xxHash/
|
||||
|
||||
## Compatibility
|
||||
|
||||
This package is in a module and the latest code is in version 2 of the module.
|
||||
You need a version of Go with at least "minimal module compatibility" to use
|
||||
github.com/cespare/xxhash/v2:
|
||||
|
||||
* 1.9.7+ for Go 1.9
|
||||
* 1.10.3+ for Go 1.10
|
||||
* Go 1.11 or later
|
||||
|
||||
I recommend using the latest release of Go.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Here are some quick benchmarks comparing the pure-Go and assembly
|
||||
implementations of Sum64.
|
||||
|
||||
| input size | purego | asm |
|
||||
| ---------- | --------- | --------- |
|
||||
| 4 B | 1.3 GB/s | 1.2 GB/s |
|
||||
| 16 B | 2.9 GB/s | 3.5 GB/s |
|
||||
| 100 B | 6.9 GB/s | 8.1 GB/s |
|
||||
| 4 KB | 11.7 GB/s | 16.7 GB/s |
|
||||
| 10 MB | 12.0 GB/s | 17.3 GB/s |
|
||||
|
||||
These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
|
||||
CPU using the following commands under Go 1.19.2:
|
||||
|
||||
```
|
||||
benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
|
||||
benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
|
||||
```
|
||||
|
||||
## Projects using this package
|
||||
|
||||
- [InfluxDB](https://github.com/influxdata/influxdb)
|
||||
- [Prometheus](https://github.com/prometheus/prometheus)
|
||||
- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
|
||||
- [FreeCache](https://github.com/coocood/freecache)
|
||||
- [FastCache](https://github.com/VictoriaMetrics/fastcache)
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -eu -o pipefail
|
||||
|
||||
# Small convenience script for running the tests with various combinations of
|
||||
# arch/tags. This assumes we're running on amd64 and have qemu available.
|
||||
|
||||
go test ./...
|
||||
go test -tags purego ./...
|
||||
GOARCH=arm64 go test
|
||||
GOARCH=arm64 go test -tags purego
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
|
||||
// at http://cyan4973.github.io/xxHash/.
|
||||
package xxhash
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
prime1 uint64 = 11400714785074694791
|
||||
prime2 uint64 = 14029467366897019727
|
||||
prime3 uint64 = 1609587929392839161
|
||||
prime4 uint64 = 9650029242287828579
|
||||
prime5 uint64 = 2870177450012600261
|
||||
)
|
||||
|
||||
// Store the primes in an array as well.
|
||||
//
|
||||
// The consts are used when possible in Go code to avoid MOVs but we need a
|
||||
// contiguous array of the assembly code.
|
||||
var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
|
||||
|
||||
// Digest implements hash.Hash64.
|
||||
type Digest struct {
|
||||
v1 uint64
|
||||
v2 uint64
|
||||
v3 uint64
|
||||
v4 uint64
|
||||
total uint64
|
||||
mem [32]byte
|
||||
n int // how much of mem is used
|
||||
}
|
||||
|
||||
// New creates a new Digest that computes the 64-bit xxHash algorithm.
|
||||
func New() *Digest {
|
||||
var d Digest
|
||||
d.Reset()
|
||||
return &d
|
||||
}
|
||||
|
||||
// Reset clears the Digest's state so that it can be reused.
|
||||
func (d *Digest) Reset() {
|
||||
d.v1 = primes[0] + prime2
|
||||
d.v2 = prime2
|
||||
d.v3 = 0
|
||||
d.v4 = -primes[0]
|
||||
d.total = 0
|
||||
d.n = 0
|
||||
}
|
||||
|
||||
// Size always returns 8 bytes.
|
||||
func (d *Digest) Size() int { return 8 }
|
||||
|
||||
// BlockSize always returns 32 bytes.
|
||||
func (d *Digest) BlockSize() int { return 32 }
|
||||
|
||||
// Write adds more data to d. It always returns len(b), nil.
|
||||
func (d *Digest) Write(b []byte) (n int, err error) {
|
||||
n = len(b)
|
||||
d.total += uint64(n)
|
||||
|
||||
memleft := d.mem[d.n&(len(d.mem)-1):]
|
||||
|
||||
if d.n+n < 32 {
|
||||
// This new data doesn't even fill the current block.
|
||||
copy(memleft, b)
|
||||
d.n += n
|
||||
return
|
||||
}
|
||||
|
||||
if d.n > 0 {
|
||||
// Finish off the partial block.
|
||||
c := copy(memleft, b)
|
||||
d.v1 = round(d.v1, u64(d.mem[0:8]))
|
||||
d.v2 = round(d.v2, u64(d.mem[8:16]))
|
||||
d.v3 = round(d.v3, u64(d.mem[16:24]))
|
||||
d.v4 = round(d.v4, u64(d.mem[24:32]))
|
||||
b = b[c:]
|
||||
d.n = 0
|
||||
}
|
||||
|
||||
if len(b) >= 32 {
|
||||
// One or more full blocks left.
|
||||
nw := writeBlocks(d, b)
|
||||
b = b[nw:]
|
||||
}
|
||||
|
||||
// Store any remaining partial block.
|
||||
copy(d.mem[:], b)
|
||||
d.n = len(b)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Sum appends the current hash to b and returns the resulting slice.
|
||||
func (d *Digest) Sum(b []byte) []byte {
|
||||
s := d.Sum64()
|
||||
return append(
|
||||
b,
|
||||
byte(s>>56),
|
||||
byte(s>>48),
|
||||
byte(s>>40),
|
||||
byte(s>>32),
|
||||
byte(s>>24),
|
||||
byte(s>>16),
|
||||
byte(s>>8),
|
||||
byte(s),
|
||||
)
|
||||
}
|
||||
|
||||
// Sum64 returns the current hash.
|
||||
func (d *Digest) Sum64() uint64 {
|
||||
var h uint64
|
||||
|
||||
if d.total >= 32 {
|
||||
v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
|
||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
||||
h = mergeRound(h, v1)
|
||||
h = mergeRound(h, v2)
|
||||
h = mergeRound(h, v3)
|
||||
h = mergeRound(h, v4)
|
||||
} else {
|
||||
h = d.v3 + prime5
|
||||
}
|
||||
|
||||
h += d.total
|
||||
|
||||
b := d.mem[:d.n&(len(d.mem)-1)]
|
||||
for ; len(b) >= 8; b = b[8:] {
|
||||
k1 := round(0, u64(b[:8]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if len(b) >= 4 {
|
||||
h ^= uint64(u32(b[:4])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
b = b[4:]
|
||||
}
|
||||
for ; len(b) > 0; b = b[1:] {
|
||||
h ^= uint64(b[0]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
}
|
||||
|
||||
h ^= h >> 33
|
||||
h *= prime2
|
||||
h ^= h >> 29
|
||||
h *= prime3
|
||||
h ^= h >> 32
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
const (
|
||||
magic = "xxh\x06"
|
||||
marshaledSize = len(magic) + 8*5 + 32
|
||||
)
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface.
|
||||
func (d *Digest) MarshalBinary() ([]byte, error) {
|
||||
b := make([]byte, 0, marshaledSize)
|
||||
b = append(b, magic...)
|
||||
b = appendUint64(b, d.v1)
|
||||
b = appendUint64(b, d.v2)
|
||||
b = appendUint64(b, d.v3)
|
||||
b = appendUint64(b, d.v4)
|
||||
b = appendUint64(b, d.total)
|
||||
b = append(b, d.mem[:d.n]...)
|
||||
b = b[:len(b)+len(d.mem)-d.n]
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
|
||||
func (d *Digest) UnmarshalBinary(b []byte) error {
|
||||
if len(b) < len(magic) || string(b[:len(magic)]) != magic {
|
||||
return errors.New("xxhash: invalid hash state identifier")
|
||||
}
|
||||
if len(b) != marshaledSize {
|
||||
return errors.New("xxhash: invalid hash state size")
|
||||
}
|
||||
b = b[len(magic):]
|
||||
b, d.v1 = consumeUint64(b)
|
||||
b, d.v2 = consumeUint64(b)
|
||||
b, d.v3 = consumeUint64(b)
|
||||
b, d.v4 = consumeUint64(b)
|
||||
b, d.total = consumeUint64(b)
|
||||
copy(d.mem[:], b)
|
||||
d.n = int(d.total % uint64(len(d.mem)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendUint64(b []byte, x uint64) []byte {
|
||||
var a [8]byte
|
||||
binary.LittleEndian.PutUint64(a[:], x)
|
||||
return append(b, a[:]...)
|
||||
}
|
||||
|
||||
func consumeUint64(b []byte) ([]byte, uint64) {
|
||||
x := u64(b)
|
||||
return b[8:], x
|
||||
}
|
||||
|
||||
func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
|
||||
func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
|
||||
|
||||
func round(acc, input uint64) uint64 {
|
||||
acc += input * prime2
|
||||
acc = rol31(acc)
|
||||
acc *= prime1
|
||||
return acc
|
||||
}
|
||||
|
||||
func mergeRound(acc, val uint64) uint64 {
|
||||
val = round(0, val)
|
||||
acc ^= val
|
||||
acc = acc*prime1 + prime4
|
||||
return acc
|
||||
}
|
||||
|
||||
func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
|
||||
func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
|
||||
func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
|
||||
func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
|
||||
func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
|
||||
func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
|
||||
func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
|
||||
func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
//go:build !appengine && gc && !purego
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Registers:
|
||||
#define h AX
|
||||
#define d AX
|
||||
#define p SI // pointer to advance through b
|
||||
#define n DX
|
||||
#define end BX // loop end
|
||||
#define v1 R8
|
||||
#define v2 R9
|
||||
#define v3 R10
|
||||
#define v4 R11
|
||||
#define x R12
|
||||
#define prime1 R13
|
||||
#define prime2 R14
|
||||
#define prime4 DI
|
||||
|
||||
#define round(acc, x) \
|
||||
IMULQ prime2, x \
|
||||
ADDQ x, acc \
|
||||
ROLQ $31, acc \
|
||||
IMULQ prime1, acc
|
||||
|
||||
// round0 performs the operation x = round(0, x).
|
||||
#define round0(x) \
|
||||
IMULQ prime2, x \
|
||||
ROLQ $31, x \
|
||||
IMULQ prime1, x
|
||||
|
||||
// mergeRound applies a merge round on the two registers acc and x.
|
||||
// It assumes that prime1, prime2, and prime4 have been loaded.
|
||||
#define mergeRound(acc, x) \
|
||||
round0(x) \
|
||||
XORQ x, acc \
|
||||
IMULQ prime1, acc \
|
||||
ADDQ prime4, acc
|
||||
|
||||
// blockLoop processes as many 32-byte blocks as possible,
|
||||
// updating v1, v2, v3, and v4. It assumes that there is at least one block
|
||||
// to process.
|
||||
#define blockLoop() \
|
||||
loop: \
|
||||
MOVQ +0(p), x \
|
||||
round(v1, x) \
|
||||
MOVQ +8(p), x \
|
||||
round(v2, x) \
|
||||
MOVQ +16(p), x \
|
||||
round(v3, x) \
|
||||
MOVQ +24(p), x \
|
||||
round(v4, x) \
|
||||
ADDQ $32, p \
|
||||
CMPQ p, end \
|
||||
JLE loop
|
||||
|
||||
// func Sum64(b []byte) uint64
|
||||
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
|
||||
// Load fixed primes.
|
||||
MOVQ ·primes+0(SB), prime1
|
||||
MOVQ ·primes+8(SB), prime2
|
||||
MOVQ ·primes+24(SB), prime4
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+0(FP), p
|
||||
MOVQ b_len+8(FP), n
|
||||
LEAQ (p)(n*1), end
|
||||
|
||||
// The first loop limit will be len(b)-32.
|
||||
SUBQ $32, end
|
||||
|
||||
// Check whether we have at least one block.
|
||||
CMPQ n, $32
|
||||
JLT noBlocks
|
||||
|
||||
// Set up initial state (v1, v2, v3, v4).
|
||||
MOVQ prime1, v1
|
||||
ADDQ prime2, v1
|
||||
MOVQ prime2, v2
|
||||
XORQ v3, v3
|
||||
XORQ v4, v4
|
||||
SUBQ prime1, v4
|
||||
|
||||
blockLoop()
|
||||
|
||||
MOVQ v1, h
|
||||
ROLQ $1, h
|
||||
MOVQ v2, x
|
||||
ROLQ $7, x
|
||||
ADDQ x, h
|
||||
MOVQ v3, x
|
||||
ROLQ $12, x
|
||||
ADDQ x, h
|
||||
MOVQ v4, x
|
||||
ROLQ $18, x
|
||||
ADDQ x, h
|
||||
|
||||
mergeRound(h, v1)
|
||||
mergeRound(h, v2)
|
||||
mergeRound(h, v3)
|
||||
mergeRound(h, v4)
|
||||
|
||||
JMP afterBlocks
|
||||
|
||||
noBlocks:
|
||||
MOVQ ·primes+32(SB), h
|
||||
|
||||
afterBlocks:
|
||||
ADDQ n, h
|
||||
|
||||
ADDQ $24, end
|
||||
CMPQ p, end
|
||||
JG try4
|
||||
|
||||
loop8:
|
||||
MOVQ (p), x
|
||||
ADDQ $8, p
|
||||
round0(x)
|
||||
XORQ x, h
|
||||
ROLQ $27, h
|
||||
IMULQ prime1, h
|
||||
ADDQ prime4, h
|
||||
|
||||
CMPQ p, end
|
||||
JLE loop8
|
||||
|
||||
try4:
|
||||
ADDQ $4, end
|
||||
CMPQ p, end
|
||||
JG try1
|
||||
|
||||
MOVL (p), x
|
||||
ADDQ $4, p
|
||||
IMULQ prime1, x
|
||||
XORQ x, h
|
||||
|
||||
ROLQ $23, h
|
||||
IMULQ prime2, h
|
||||
ADDQ ·primes+16(SB), h
|
||||
|
||||
try1:
|
||||
ADDQ $4, end
|
||||
CMPQ p, end
|
||||
JGE finalize
|
||||
|
||||
loop1:
|
||||
MOVBQZX (p), x
|
||||
ADDQ $1, p
|
||||
IMULQ ·primes+32(SB), x
|
||||
XORQ x, h
|
||||
ROLQ $11, h
|
||||
IMULQ prime1, h
|
||||
|
||||
CMPQ p, end
|
||||
JL loop1
|
||||
|
||||
finalize:
|
||||
MOVQ h, x
|
||||
SHRQ $33, x
|
||||
XORQ x, h
|
||||
IMULQ prime2, h
|
||||
MOVQ h, x
|
||||
SHRQ $29, x
|
||||
XORQ x, h
|
||||
IMULQ ·primes+16(SB), h
|
||||
MOVQ h, x
|
||||
SHRQ $32, x
|
||||
XORQ x, h
|
||||
|
||||
MOVQ h, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func writeBlocks(d *Digest, b []byte) int
|
||||
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
|
||||
// Load fixed primes needed for round.
|
||||
MOVQ ·primes+0(SB), prime1
|
||||
MOVQ ·primes+8(SB), prime2
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+8(FP), p
|
||||
MOVQ b_len+16(FP), n
|
||||
LEAQ (p)(n*1), end
|
||||
SUBQ $32, end
|
||||
|
||||
// Load vN from d.
|
||||
MOVQ s+0(FP), d
|
||||
MOVQ 0(d), v1
|
||||
MOVQ 8(d), v2
|
||||
MOVQ 16(d), v3
|
||||
MOVQ 24(d), v4
|
||||
|
||||
// We don't need to check the loop condition here; this function is
|
||||
// always called with at least one block of data to process.
|
||||
blockLoop()
|
||||
|
||||
// Copy vN back to d.
|
||||
MOVQ v1, 0(d)
|
||||
MOVQ v2, 8(d)
|
||||
MOVQ v3, 16(d)
|
||||
MOVQ v4, 24(d)
|
||||
|
||||
// The number of bytes written is p minus the old base pointer.
|
||||
SUBQ b_base+8(FP), p
|
||||
MOVQ p, ret+32(FP)
|
||||
|
||||
RET
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
//go:build !appengine && gc && !purego
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Registers:
|
||||
#define digest R1
|
||||
#define h R2 // return value
|
||||
#define p R3 // input pointer
|
||||
#define n R4 // input length
|
||||
#define nblocks R5 // n / 32
|
||||
#define prime1 R7
|
||||
#define prime2 R8
|
||||
#define prime3 R9
|
||||
#define prime4 R10
|
||||
#define prime5 R11
|
||||
#define v1 R12
|
||||
#define v2 R13
|
||||
#define v3 R14
|
||||
#define v4 R15
|
||||
#define x1 R20
|
||||
#define x2 R21
|
||||
#define x3 R22
|
||||
#define x4 R23
|
||||
|
||||
#define round(acc, x) \
|
||||
MADD prime2, acc, x, acc \
|
||||
ROR $64-31, acc \
|
||||
MUL prime1, acc
|
||||
|
||||
// round0 performs the operation x = round(0, x).
|
||||
#define round0(x) \
|
||||
MUL prime2, x \
|
||||
ROR $64-31, x \
|
||||
MUL prime1, x
|
||||
|
||||
#define mergeRound(acc, x) \
|
||||
round0(x) \
|
||||
EOR x, acc \
|
||||
MADD acc, prime4, prime1, acc
|
||||
|
||||
// blockLoop processes as many 32-byte blocks as possible,
|
||||
// updating v1, v2, v3, and v4. It assumes that n >= 32.
|
||||
#define blockLoop() \
|
||||
LSR $5, n, nblocks \
|
||||
PCALIGN $16 \
|
||||
loop: \
|
||||
LDP.P 16(p), (x1, x2) \
|
||||
LDP.P 16(p), (x3, x4) \
|
||||
round(v1, x1) \
|
||||
round(v2, x2) \
|
||||
round(v3, x3) \
|
||||
round(v4, x4) \
|
||||
SUB $1, nblocks \
|
||||
CBNZ nblocks, loop
|
||||
|
||||
// func Sum64(b []byte) uint64
|
||||
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
|
||||
LDP b_base+0(FP), (p, n)
|
||||
|
||||
LDP ·primes+0(SB), (prime1, prime2)
|
||||
LDP ·primes+16(SB), (prime3, prime4)
|
||||
MOVD ·primes+32(SB), prime5
|
||||
|
||||
CMP $32, n
|
||||
CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
|
||||
BLT afterLoop
|
||||
|
||||
ADD prime1, prime2, v1
|
||||
MOVD prime2, v2
|
||||
MOVD $0, v3
|
||||
NEG prime1, v4
|
||||
|
||||
blockLoop()
|
||||
|
||||
ROR $64-1, v1, x1
|
||||
ROR $64-7, v2, x2
|
||||
ADD x1, x2
|
||||
ROR $64-12, v3, x3
|
||||
ROR $64-18, v4, x4
|
||||
ADD x3, x4
|
||||
ADD x2, x4, h
|
||||
|
||||
mergeRound(h, v1)
|
||||
mergeRound(h, v2)
|
||||
mergeRound(h, v3)
|
||||
mergeRound(h, v4)
|
||||
|
||||
afterLoop:
|
||||
ADD n, h
|
||||
|
||||
TBZ $4, n, try8
|
||||
LDP.P 16(p), (x1, x2)
|
||||
|
||||
round0(x1)
|
||||
|
||||
// NOTE: here and below, sequencing the EOR after the ROR (using a
|
||||
// rotated register) is worth a small but measurable speedup for small
|
||||
// inputs.
|
||||
ROR $64-27, h
|
||||
EOR x1 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
round0(x2)
|
||||
ROR $64-27, h
|
||||
EOR x2 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
try8:
|
||||
TBZ $3, n, try4
|
||||
MOVD.P 8(p), x1
|
||||
|
||||
round0(x1)
|
||||
ROR $64-27, h
|
||||
EOR x1 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
try4:
|
||||
TBZ $2, n, try2
|
||||
MOVWU.P 4(p), x2
|
||||
|
||||
MUL prime1, x2
|
||||
ROR $64-23, h
|
||||
EOR x2 @> 64-23, h, h
|
||||
MADD h, prime3, prime2, h
|
||||
|
||||
try2:
|
||||
TBZ $1, n, try1
|
||||
MOVHU.P 2(p), x3
|
||||
AND $255, x3, x1
|
||||
LSR $8, x3, x2
|
||||
|
||||
MUL prime5, x1
|
||||
ROR $64-11, h
|
||||
EOR x1 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
MUL prime5, x2
|
||||
ROR $64-11, h
|
||||
EOR x2 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
try1:
|
||||
TBZ $0, n, finalize
|
||||
MOVBU (p), x4
|
||||
|
||||
MUL prime5, x4
|
||||
ROR $64-11, h
|
||||
EOR x4 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
finalize:
|
||||
EOR h >> 33, h
|
||||
MUL prime2, h
|
||||
EOR h >> 29, h
|
||||
MUL prime3, h
|
||||
EOR h >> 32, h
|
||||
|
||||
MOVD h, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func writeBlocks(d *Digest, b []byte) int
|
||||
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
|
||||
LDP ·primes+0(SB), (prime1, prime2)
|
||||
|
||||
// Load state. Assume v[1-4] are stored contiguously.
|
||||
MOVD d+0(FP), digest
|
||||
LDP 0(digest), (v1, v2)
|
||||
LDP 16(digest), (v3, v4)
|
||||
|
||||
LDP b_base+8(FP), (p, n)
|
||||
|
||||
blockLoop()
|
||||
|
||||
// Store updated state.
|
||||
STP (v1, v2), 0(digest)
|
||||
STP (v3, v4), 16(digest)
|
||||
|
||||
BIC $31, n
|
||||
MOVD n, ret+32(FP)
|
||||
RET
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
//go:build (amd64 || arm64) && !appengine && gc && !purego
|
||||
// +build amd64 arm64
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64 computes the 64-bit xxHash digest of b.
|
||||
//
|
||||
//go:noescape
|
||||
func Sum64(b []byte) uint64
|
||||
|
||||
//go:noescape
|
||||
func writeBlocks(d *Digest, b []byte) int
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
//go:build (!amd64 && !arm64) || appengine || !gc || purego
|
||||
// +build !amd64,!arm64 appengine !gc purego
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64 computes the 64-bit xxHash digest of b.
|
||||
func Sum64(b []byte) uint64 {
|
||||
// A simpler version would be
|
||||
// d := New()
|
||||
// d.Write(b)
|
||||
// return d.Sum64()
|
||||
// but this is faster, particularly for small inputs.
|
||||
|
||||
n := len(b)
|
||||
var h uint64
|
||||
|
||||
if n >= 32 {
|
||||
v1 := primes[0] + prime2
|
||||
v2 := prime2
|
||||
v3 := uint64(0)
|
||||
v4 := -primes[0]
|
||||
for len(b) >= 32 {
|
||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
||||
b = b[32:len(b):len(b)]
|
||||
}
|
||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
||||
h = mergeRound(h, v1)
|
||||
h = mergeRound(h, v2)
|
||||
h = mergeRound(h, v3)
|
||||
h = mergeRound(h, v4)
|
||||
} else {
|
||||
h = prime5
|
||||
}
|
||||
|
||||
h += uint64(n)
|
||||
|
||||
for ; len(b) >= 8; b = b[8:] {
|
||||
k1 := round(0, u64(b[:8]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if len(b) >= 4 {
|
||||
h ^= uint64(u32(b[:4])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
b = b[4:]
|
||||
}
|
||||
for ; len(b) > 0; b = b[1:] {
|
||||
h ^= uint64(b[0]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
}
|
||||
|
||||
h ^= h >> 33
|
||||
h *= prime2
|
||||
h ^= h >> 29
|
||||
h *= prime3
|
||||
h ^= h >> 32
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func writeBlocks(d *Digest, b []byte) int {
|
||||
v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
|
||||
n := len(b)
|
||||
for len(b) >= 32 {
|
||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
||||
b = b[32:len(b):len(b)]
|
||||
}
|
||||
d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4
|
||||
return n - len(b)
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
//go:build appengine
|
||||
// +build appengine
|
||||
|
||||
// This file contains the safe implementations of otherwise unsafe-using code.
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64String computes the 64-bit xxHash digest of s.
|
||||
func Sum64String(s string) uint64 {
|
||||
return Sum64([]byte(s))
|
||||
}
|
||||
|
||||
// WriteString adds more data to d. It always returns len(s), nil.
|
||||
func (d *Digest) WriteString(s string) (n int, err error) {
|
||||
return d.Write([]byte(s))
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
//go:build !appengine
|
||||
// +build !appengine
|
||||
|
||||
// This file encapsulates usage of unsafe.
|
||||
// xxhash_safe.go contains the safe implementations.
|
||||
|
||||
package xxhash
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// In the future it's possible that compiler optimizations will make these
|
||||
// XxxString functions unnecessary by realizing that calls such as
|
||||
// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205.
|
||||
// If that happens, even if we keep these functions they can be replaced with
|
||||
// the trivial safe code.
|
||||
|
||||
// NOTE: The usual way of doing an unsafe string-to-[]byte conversion is:
|
||||
//
|
||||
// var b []byte
|
||||
// bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
// bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
|
||||
// bh.Len = len(s)
|
||||
// bh.Cap = len(s)
|
||||
//
|
||||
// Unfortunately, as of Go 1.15.3 the inliner's cost model assigns a high enough
|
||||
// weight to this sequence of expressions that any function that uses it will
|
||||
// not be inlined. Instead, the functions below use a different unsafe
|
||||
// conversion designed to minimize the inliner weight and allow both to be
|
||||
// inlined. There is also a test (TestInlining) which verifies that these are
|
||||
// inlined.
|
||||
//
|
||||
// See https://github.com/golang/go/issues/42739 for discussion.
|
||||
|
||||
// Sum64String computes the 64-bit xxHash digest of s.
|
||||
// It may be faster than Sum64([]byte(s)) by avoiding a copy.
|
||||
func Sum64String(s string) uint64 {
|
||||
b := *(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)}))
|
||||
return Sum64(b)
|
||||
}
|
||||
|
||||
// WriteString adds more data to d. It always returns len(s), nil.
|
||||
// It may be faster than Write([]byte(s)) by avoiding a copy.
|
||||
func (d *Digest) WriteString(s string) (n int, err error) {
|
||||
d.Write(*(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)})))
|
||||
// d.Write always returns len(s), nil.
|
||||
// Ignoring the return output and returning these fixed values buys a
|
||||
// savings of 6 in the inliner's cost model.
|
||||
return len(s), nil
|
||||
}
|
||||
|
||||
// sliceHeader is similar to reflect.SliceHeader, but it assumes that the layout
|
||||
// of the first two words is the same as the layout of a string.
|
||||
type sliceHeader struct {
|
||||
s string
|
||||
cap int
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.6
|
||||
- 1.7
|
||||
- 1.8
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
.PHONY: ci generate clean
|
||||
|
||||
ci: clean generate
|
||||
go test -v ./...
|
||||
|
||||
generate:
|
||||
go generate .
|
||||
|
||||
clean:
|
||||
rm -rf *_generated*.go
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
# httpsnoop
|
||||
|
||||
Package httpsnoop provides an easy way to capture http related metrics (i.e.
|
||||
response time, bytes written, and http status code) from your application's
|
||||
http.Handlers.
|
||||
|
||||
Doing this requires non-trivial wrapping of the http.ResponseWriter interface,
|
||||
which is also exposed for users interested in a more low-level API.
|
||||
|
||||
[](https://godoc.org/github.com/felixge/httpsnoop)
|
||||
[](https://travis-ci.org/felixge/httpsnoop)
|
||||
|
||||
## Usage Example
|
||||
|
||||
```go
|
||||
// myH is your app's http handler, perhaps a http.ServeMux or similar.
|
||||
var myH http.Handler
|
||||
// wrappedH wraps myH in order to log every request.
|
||||
wrappedH := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m := httpsnoop.CaptureMetrics(myH, w, r)
|
||||
log.Printf(
|
||||
"%s %s (code=%d dt=%s written=%d)",
|
||||
r.Method,
|
||||
r.URL,
|
||||
m.Code,
|
||||
m.Duration,
|
||||
m.Written,
|
||||
)
|
||||
})
|
||||
http.ListenAndServe(":8080", wrappedH)
|
||||
```
|
||||
|
||||
## Why this package exists
|
||||
|
||||
Instrumenting an application's http.Handler is surprisingly difficult.
|
||||
|
||||
However if you google for e.g. "capture ResponseWriter status code" you'll find
|
||||
lots of advise and code examples that suggest it to be a fairly trivial
|
||||
undertaking. Unfortunately everything I've seen so far has a high chance of
|
||||
breaking your application.
|
||||
|
||||
The main problem is that a `http.ResponseWriter` often implements additional
|
||||
interfaces such as `http.Flusher`, `http.CloseNotifier`, `http.Hijacker`, `http.Pusher`, and
|
||||
`io.ReaderFrom`. So the naive approach of just wrapping `http.ResponseWriter`
|
||||
in your own struct that also implements the `http.ResponseWriter` interface
|
||||
will hide the additional interfaces mentioned above. This has a high change of
|
||||
introducing subtle bugs into any non-trivial application.
|
||||
|
||||
Another approach I've seen people take is to return a struct that implements
|
||||
all of the interfaces above. However, that's also problematic, because it's
|
||||
difficult to fake some of these interfaces behaviors when the underlying
|
||||
`http.ResponseWriter` doesn't have an implementation. It's also dangerous,
|
||||
because an application may choose to operate differently, merely because it
|
||||
detects the presence of these additional interfaces.
|
||||
|
||||
This package solves this problem by checking which additional interfaces a
|
||||
`http.ResponseWriter` implements, returning a wrapped version implementing the
|
||||
exact same set of interfaces.
|
||||
|
||||
Additionally this package properly handles edge cases such as `WriteHeader` not
|
||||
being called, or called more than once, as well as concurrent calls to
|
||||
`http.ResponseWriter` methods, and even calls happening after the wrapped
|
||||
`ServeHTTP` has already returned.
|
||||
|
||||
Unfortunately this package is not perfect either. It's possible that it is
|
||||
still missing some interfaces provided by the go core (let me know if you find
|
||||
one), and it won't work for applications adding their own interfaces into the
|
||||
mix. You can however use `httpsnoop.Unwrap(w)` to access the underlying
|
||||
`http.ResponseWriter` and type-assert the result to its other interfaces.
|
||||
|
||||
However, hopefully the explanation above has sufficiently scared you of rolling
|
||||
your own solution to this problem. httpsnoop may still break your application,
|
||||
but at least it tries to avoid it as much as possible.
|
||||
|
||||
Anyway, the real problem here is that smuggling additional interfaces inside
|
||||
`http.ResponseWriter` is a problematic design choice, but it probably goes as
|
||||
deep as the Go language specification itself. But that's okay, I still prefer
|
||||
Go over the alternatives ;).
|
||||
|
||||
## Performance
|
||||
|
||||
```
|
||||
BenchmarkBaseline-8 20000 94912 ns/op
|
||||
BenchmarkCaptureMetrics-8 20000 95461 ns/op
|
||||
```
|
||||
|
||||
As you can see, using `CaptureMetrics` on a vanilla http.Handler introduces an
|
||||
overhead of ~500 ns per http request on my machine. However, the margin of
|
||||
error appears to be larger than that, therefor it should be reasonable to
|
||||
assume that the overhead introduced by `CaptureMetrics` is absolutely
|
||||
negligible.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package httpsnoop
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Metrics holds metrics captured from CaptureMetrics.
|
||||
type Metrics struct {
|
||||
// Code is the first http response code passed to the WriteHeader func of
|
||||
// the ResponseWriter. If no such call is made, a default code of 200 is
|
||||
// assumed instead.
|
||||
Code int
|
||||
// Duration is the time it took to execute the handler.
|
||||
Duration time.Duration
|
||||
// Written is the number of bytes successfully written by the Write or
|
||||
// ReadFrom function of the ResponseWriter. ResponseWriters may also write
|
||||
// data to their underlaying connection directly (e.g. headers), but those
|
||||
// are not tracked. Therefor the number of Written bytes will usually match
|
||||
// the size of the response body.
|
||||
Written int64
|
||||
}
|
||||
|
||||
// CaptureMetrics wraps the given hnd, executes it with the given w and r, and
|
||||
// returns the metrics it captured from it.
|
||||
func CaptureMetrics(hnd http.Handler, w http.ResponseWriter, r *http.Request) Metrics {
|
||||
return CaptureMetricsFn(w, func(ww http.ResponseWriter) {
|
||||
hnd.ServeHTTP(ww, r)
|
||||
})
|
||||
}
|
||||
|
||||
// CaptureMetricsFn wraps w and calls fn with the wrapped w and returns the
|
||||
// resulting metrics. This is very similar to CaptureMetrics (which is just
|
||||
// sugar on top of this func), but is a more usable interface if your
|
||||
// application doesn't use the Go http.Handler interface.
|
||||
func CaptureMetricsFn(w http.ResponseWriter, fn func(http.ResponseWriter)) Metrics {
|
||||
m := Metrics{Code: http.StatusOK}
|
||||
m.CaptureMetrics(w, fn)
|
||||
return m
|
||||
}
|
||||
|
||||
// CaptureMetrics wraps w and calls fn with the wrapped w and updates
|
||||
// Metrics m with the resulting metrics. This is similar to CaptureMetricsFn,
|
||||
// but allows one to customize starting Metrics object.
|
||||
func (m *Metrics) CaptureMetrics(w http.ResponseWriter, fn func(http.ResponseWriter)) {
|
||||
var (
|
||||
start = time.Now()
|
||||
headerWritten bool
|
||||
hooks = Hooks{
|
||||
WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc {
|
||||
return func(code int) {
|
||||
next(code)
|
||||
|
||||
if !headerWritten {
|
||||
m.Code = code
|
||||
headerWritten = true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
Write: func(next WriteFunc) WriteFunc {
|
||||
return func(p []byte) (int, error) {
|
||||
n, err := next(p)
|
||||
|
||||
m.Written += int64(n)
|
||||
headerWritten = true
|
||||
return n, err
|
||||
}
|
||||
},
|
||||
|
||||
ReadFrom: func(next ReadFromFunc) ReadFromFunc {
|
||||
return func(src io.Reader) (int64, error) {
|
||||
n, err := next(src)
|
||||
|
||||
headerWritten = true
|
||||
m.Written += n
|
||||
return n, err
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
fn(Wrap(w, hooks))
|
||||
m.Duration += time.Since(start)
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// Package httpsnoop provides an easy way to capture http related metrics (i.e.
|
||||
// response time, bytes written, and http status code) from your application's
|
||||
// http.Handlers.
|
||||
//
|
||||
// Doing this requires non-trivial wrapping of the http.ResponseWriter
|
||||
// interface, which is also exposed for users interested in a more low-level
|
||||
// API.
|
||||
package httpsnoop
|
||||
|
||||
//go:generate go run codegen/main.go
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
// +build go1.8
|
||||
// Code generated by "httpsnoop/codegen"; DO NOT EDIT
|
||||
|
||||
package httpsnoop
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// HeaderFunc is part of the http.ResponseWriter interface.
|
||||
type HeaderFunc func() http.Header
|
||||
|
||||
// WriteHeaderFunc is part of the http.ResponseWriter interface.
|
||||
type WriteHeaderFunc func(code int)
|
||||
|
||||
// WriteFunc is part of the http.ResponseWriter interface.
|
||||
type WriteFunc func(b []byte) (int, error)
|
||||
|
||||
// FlushFunc is part of the http.Flusher interface.
|
||||
type FlushFunc func()
|
||||
|
||||
// CloseNotifyFunc is part of the http.CloseNotifier interface.
|
||||
type CloseNotifyFunc func() <-chan bool
|
||||
|
||||
// HijackFunc is part of the http.Hijacker interface.
|
||||
type HijackFunc func() (net.Conn, *bufio.ReadWriter, error)
|
||||
|
||||
// ReadFromFunc is part of the io.ReaderFrom interface.
|
||||
type ReadFromFunc func(src io.Reader) (int64, error)
|
||||
|
||||
// PushFunc is part of the http.Pusher interface.
|
||||
type PushFunc func(target string, opts *http.PushOptions) error
|
||||
|
||||
// Hooks defines a set of method interceptors for methods included in
|
||||
// http.ResponseWriter as well as some others. You can think of them as
|
||||
// middleware for the function calls they target. See Wrap for more details.
|
||||
type Hooks struct {
|
||||
Header func(HeaderFunc) HeaderFunc
|
||||
WriteHeader func(WriteHeaderFunc) WriteHeaderFunc
|
||||
Write func(WriteFunc) WriteFunc
|
||||
Flush func(FlushFunc) FlushFunc
|
||||
CloseNotify func(CloseNotifyFunc) CloseNotifyFunc
|
||||
Hijack func(HijackFunc) HijackFunc
|
||||
ReadFrom func(ReadFromFunc) ReadFromFunc
|
||||
Push func(PushFunc) PushFunc
|
||||
}
|
||||
|
||||
// Wrap returns a wrapped version of w that provides the exact same interface
|
||||
// as w. Specifically if w implements any combination of:
|
||||
//
|
||||
// - http.Flusher
|
||||
// - http.CloseNotifier
|
||||
// - http.Hijacker
|
||||
// - io.ReaderFrom
|
||||
// - http.Pusher
|
||||
//
|
||||
// The wrapped version will implement the exact same combination. If no hooks
|
||||
// are set, the wrapped version also behaves exactly as w. Hooks targeting
|
||||
// methods not supported by w are ignored. Any other hooks will intercept the
|
||||
// method they target and may modify the call's arguments and/or return values.
|
||||
// The CaptureMetrics implementation serves as a working example for how the
|
||||
// hooks can be used.
|
||||
func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter {
|
||||
rw := &rw{w: w, h: hooks}
|
||||
_, i0 := w.(http.Flusher)
|
||||
_, i1 := w.(http.CloseNotifier)
|
||||
_, i2 := w.(http.Hijacker)
|
||||
_, i3 := w.(io.ReaderFrom)
|
||||
_, i4 := w.(http.Pusher)
|
||||
switch {
|
||||
// combination 1/32
|
||||
case !i0 && !i1 && !i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
}{rw, rw}
|
||||
// combination 2/32
|
||||
case !i0 && !i1 && !i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Pusher
|
||||
}{rw, rw, rw}
|
||||
// combination 3/32
|
||||
case !i0 && !i1 && !i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw}
|
||||
// combination 4/32
|
||||
case !i0 && !i1 && !i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 5/32
|
||||
case !i0 && !i1 && i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
}{rw, rw, rw}
|
||||
// combination 6/32
|
||||
case !i0 && !i1 && i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 7/32
|
||||
case !i0 && !i1 && i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 8/32
|
||||
case !i0 && !i1 && i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 9/32
|
||||
case !i0 && i1 && !i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
}{rw, rw, rw}
|
||||
// combination 10/32
|
||||
case !i0 && i1 && !i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 11/32
|
||||
case !i0 && i1 && !i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 12/32
|
||||
case !i0 && i1 && !i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 13/32
|
||||
case !i0 && i1 && i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 14/32
|
||||
case !i0 && i1 && i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 15/32
|
||||
case !i0 && i1 && i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 16/32
|
||||
case !i0 && i1 && i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
// combination 17/32
|
||||
case i0 && !i1 && !i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
}{rw, rw, rw}
|
||||
// combination 18/32
|
||||
case i0 && !i1 && !i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 19/32
|
||||
case i0 && !i1 && !i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 20/32
|
||||
case i0 && !i1 && !i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 21/32
|
||||
case i0 && !i1 && i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 22/32
|
||||
case i0 && !i1 && i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 23/32
|
||||
case i0 && !i1 && i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 24/32
|
||||
case i0 && !i1 && i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
// combination 25/32
|
||||
case i0 && i1 && !i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 26/32
|
||||
case i0 && i1 && !i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 27/32
|
||||
case i0 && i1 && !i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 28/32
|
||||
case i0 && i1 && !i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
// combination 29/32
|
||||
case i0 && i1 && i2 && !i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 30/32
|
||||
case i0 && i1 && i2 && !i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
// combination 31/32
|
||||
case i0 && i1 && i2 && i3 && !i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
// combination 32/32
|
||||
case i0 && i1 && i2 && i3 && i4:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
http.Pusher
|
||||
}{rw, rw, rw, rw, rw, rw, rw}
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
type rw struct {
|
||||
w http.ResponseWriter
|
||||
h Hooks
|
||||
}
|
||||
|
||||
func (w *rw) Unwrap() http.ResponseWriter {
|
||||
return w.w
|
||||
}
|
||||
|
||||
func (w *rw) Header() http.Header {
|
||||
f := w.w.(http.ResponseWriter).Header
|
||||
if w.h.Header != nil {
|
||||
f = w.h.Header(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) WriteHeader(code int) {
|
||||
f := w.w.(http.ResponseWriter).WriteHeader
|
||||
if w.h.WriteHeader != nil {
|
||||
f = w.h.WriteHeader(f)
|
||||
}
|
||||
f(code)
|
||||
}
|
||||
|
||||
func (w *rw) Write(b []byte) (int, error) {
|
||||
f := w.w.(http.ResponseWriter).Write
|
||||
if w.h.Write != nil {
|
||||
f = w.h.Write(f)
|
||||
}
|
||||
return f(b)
|
||||
}
|
||||
|
||||
func (w *rw) Flush() {
|
||||
f := w.w.(http.Flusher).Flush
|
||||
if w.h.Flush != nil {
|
||||
f = w.h.Flush(f)
|
||||
}
|
||||
f()
|
||||
}
|
||||
|
||||
func (w *rw) CloseNotify() <-chan bool {
|
||||
f := w.w.(http.CloseNotifier).CloseNotify
|
||||
if w.h.CloseNotify != nil {
|
||||
f = w.h.CloseNotify(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
f := w.w.(http.Hijacker).Hijack
|
||||
if w.h.Hijack != nil {
|
||||
f = w.h.Hijack(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) ReadFrom(src io.Reader) (int64, error) {
|
||||
f := w.w.(io.ReaderFrom).ReadFrom
|
||||
if w.h.ReadFrom != nil {
|
||||
f = w.h.ReadFrom(f)
|
||||
}
|
||||
return f(src)
|
||||
}
|
||||
|
||||
func (w *rw) Push(target string, opts *http.PushOptions) error {
|
||||
f := w.w.(http.Pusher).Push
|
||||
if w.h.Push != nil {
|
||||
f = w.h.Push(f)
|
||||
}
|
||||
return f(target, opts)
|
||||
}
|
||||
|
||||
type Unwrapper interface {
|
||||
Unwrap() http.ResponseWriter
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying http.ResponseWriter from within zero or more
|
||||
// layers of httpsnoop wrappers.
|
||||
func Unwrap(w http.ResponseWriter) http.ResponseWriter {
|
||||
if rw, ok := w.(Unwrapper); ok {
|
||||
// recurse until rw.Unwrap() returns a non-Unwrapper
|
||||
return Unwrap(rw.Unwrap())
|
||||
} else {
|
||||
return w
|
||||
}
|
||||
}
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
// +build !go1.8
|
||||
// Code generated by "httpsnoop/codegen"; DO NOT EDIT
|
||||
|
||||
package httpsnoop
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// HeaderFunc is part of the http.ResponseWriter interface.
|
||||
type HeaderFunc func() http.Header
|
||||
|
||||
// WriteHeaderFunc is part of the http.ResponseWriter interface.
|
||||
type WriteHeaderFunc func(code int)
|
||||
|
||||
// WriteFunc is part of the http.ResponseWriter interface.
|
||||
type WriteFunc func(b []byte) (int, error)
|
||||
|
||||
// FlushFunc is part of the http.Flusher interface.
|
||||
type FlushFunc func()
|
||||
|
||||
// CloseNotifyFunc is part of the http.CloseNotifier interface.
|
||||
type CloseNotifyFunc func() <-chan bool
|
||||
|
||||
// HijackFunc is part of the http.Hijacker interface.
|
||||
type HijackFunc func() (net.Conn, *bufio.ReadWriter, error)
|
||||
|
||||
// ReadFromFunc is part of the io.ReaderFrom interface.
|
||||
type ReadFromFunc func(src io.Reader) (int64, error)
|
||||
|
||||
// Hooks defines a set of method interceptors for methods included in
|
||||
// http.ResponseWriter as well as some others. You can think of them as
|
||||
// middleware for the function calls they target. See Wrap for more details.
|
||||
type Hooks struct {
|
||||
Header func(HeaderFunc) HeaderFunc
|
||||
WriteHeader func(WriteHeaderFunc) WriteHeaderFunc
|
||||
Write func(WriteFunc) WriteFunc
|
||||
Flush func(FlushFunc) FlushFunc
|
||||
CloseNotify func(CloseNotifyFunc) CloseNotifyFunc
|
||||
Hijack func(HijackFunc) HijackFunc
|
||||
ReadFrom func(ReadFromFunc) ReadFromFunc
|
||||
}
|
||||
|
||||
// Wrap returns a wrapped version of w that provides the exact same interface
|
||||
// as w. Specifically if w implements any combination of:
|
||||
//
|
||||
// - http.Flusher
|
||||
// - http.CloseNotifier
|
||||
// - http.Hijacker
|
||||
// - io.ReaderFrom
|
||||
//
|
||||
// The wrapped version will implement the exact same combination. If no hooks
|
||||
// are set, the wrapped version also behaves exactly as w. Hooks targeting
|
||||
// methods not supported by w are ignored. Any other hooks will intercept the
|
||||
// method they target and may modify the call's arguments and/or return values.
|
||||
// The CaptureMetrics implementation serves as a working example for how the
|
||||
// hooks can be used.
|
||||
func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter {
|
||||
rw := &rw{w: w, h: hooks}
|
||||
_, i0 := w.(http.Flusher)
|
||||
_, i1 := w.(http.CloseNotifier)
|
||||
_, i2 := w.(http.Hijacker)
|
||||
_, i3 := w.(io.ReaderFrom)
|
||||
switch {
|
||||
// combination 1/16
|
||||
case !i0 && !i1 && !i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
}{rw, rw}
|
||||
// combination 2/16
|
||||
case !i0 && !i1 && !i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw}
|
||||
// combination 3/16
|
||||
case !i0 && !i1 && i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
}{rw, rw, rw}
|
||||
// combination 4/16
|
||||
case !i0 && !i1 && i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 5/16
|
||||
case !i0 && i1 && !i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
}{rw, rw, rw}
|
||||
// combination 6/16
|
||||
case !i0 && i1 && !i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 7/16
|
||||
case !i0 && i1 && i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 8/16
|
||||
case !i0 && i1 && i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 9/16
|
||||
case i0 && !i1 && !i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
}{rw, rw, rw}
|
||||
// combination 10/16
|
||||
case i0 && !i1 && !i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 11/16
|
||||
case i0 && !i1 && i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 12/16
|
||||
case i0 && !i1 && i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 13/16
|
||||
case i0 && i1 && !i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
}{rw, rw, rw, rw}
|
||||
// combination 14/16
|
||||
case i0 && i1 && !i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 15/16
|
||||
case i0 && i1 && i2 && !i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
}{rw, rw, rw, rw, rw}
|
||||
// combination 16/16
|
||||
case i0 && i1 && i2 && i3:
|
||||
return struct {
|
||||
Unwrapper
|
||||
http.ResponseWriter
|
||||
http.Flusher
|
||||
http.CloseNotifier
|
||||
http.Hijacker
|
||||
io.ReaderFrom
|
||||
}{rw, rw, rw, rw, rw, rw}
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
type rw struct {
|
||||
w http.ResponseWriter
|
||||
h Hooks
|
||||
}
|
||||
|
||||
func (w *rw) Unwrap() http.ResponseWriter {
|
||||
return w.w
|
||||
}
|
||||
|
||||
func (w *rw) Header() http.Header {
|
||||
f := w.w.(http.ResponseWriter).Header
|
||||
if w.h.Header != nil {
|
||||
f = w.h.Header(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) WriteHeader(code int) {
|
||||
f := w.w.(http.ResponseWriter).WriteHeader
|
||||
if w.h.WriteHeader != nil {
|
||||
f = w.h.WriteHeader(f)
|
||||
}
|
||||
f(code)
|
||||
}
|
||||
|
||||
func (w *rw) Write(b []byte) (int, error) {
|
||||
f := w.w.(http.ResponseWriter).Write
|
||||
if w.h.Write != nil {
|
||||
f = w.h.Write(f)
|
||||
}
|
||||
return f(b)
|
||||
}
|
||||
|
||||
func (w *rw) Flush() {
|
||||
f := w.w.(http.Flusher).Flush
|
||||
if w.h.Flush != nil {
|
||||
f = w.h.Flush(f)
|
||||
}
|
||||
f()
|
||||
}
|
||||
|
||||
func (w *rw) CloseNotify() <-chan bool {
|
||||
f := w.w.(http.CloseNotifier).CloseNotify
|
||||
if w.h.CloseNotify != nil {
|
||||
f = w.h.CloseNotify(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
f := w.w.(http.Hijacker).Hijack
|
||||
if w.h.Hijack != nil {
|
||||
f = w.h.Hijack(f)
|
||||
}
|
||||
return f()
|
||||
}
|
||||
|
||||
func (w *rw) ReadFrom(src io.Reader) (int64, error) {
|
||||
f := w.w.(io.ReaderFrom).ReadFrom
|
||||
if w.h.ReadFrom != nil {
|
||||
f = w.h.ReadFrom(f)
|
||||
}
|
||||
return f(src)
|
||||
}
|
||||
|
||||
type Unwrapper interface {
|
||||
Unwrap() http.ResponseWriter
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying http.ResponseWriter from within zero or more
|
||||
// layers of httpsnoop wrappers.
|
||||
func Unwrap(w http.ResponseWriter) http.ResponseWriter {
|
||||
if rw, ok := w.(Unwrapper); ok {
|
||||
// recurse until rw.Unwrap() returns a non-Unwrapper
|
||||
return Unwrap(rw.Unwrap())
|
||||
} else {
|
||||
return w
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Peter Bourgon
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
// Package endpoint defines an abstraction for RPCs.
|
||||
//
|
||||
// Endpoints are a fundamental building block for many Go kit components.
|
||||
// Endpoints are implemented by servers, and called by clients.
|
||||
package endpoint
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// Endpoint is the fundamental building block of servers and clients.
|
||||
// It represents a single RPC method.
|
||||
type Endpoint func(ctx context.Context, request interface{}) (response interface{}, err error)
|
||||
|
||||
// Nop is an endpoint that does nothing and returns a nil error.
|
||||
// Useful for tests.
|
||||
func Nop(context.Context, interface{}) (interface{}, error) { return struct{}{}, nil }
|
||||
|
||||
// Middleware is a chainable behavior modifier for endpoints.
|
||||
type Middleware func(Endpoint) Endpoint
|
||||
|
||||
// Chain is a helper function for composing middlewares. Requests will
|
||||
// traverse them in the order they're declared. That is, the first middleware
|
||||
// is treated as the outermost middleware.
|
||||
func Chain(outer Middleware, others ...Middleware) Middleware {
|
||||
return func(next Endpoint) Endpoint {
|
||||
for i := len(others) - 1; i >= 0; i-- { // reverse
|
||||
next = others[i](next)
|
||||
}
|
||||
return outer(next)
|
||||
}
|
||||
}
|
||||
|
||||
// Failer may be implemented by Go kit response types that contain business
|
||||
// logic error details. If Failed returns a non-nil error, the Go kit transport
|
||||
// layer may interpret this as a business logic error, and may encode it
|
||||
// differently than a regular, successful response.
|
||||
//
|
||||
// It's not necessary for your response types to implement Failer, but it may
|
||||
// help for more sophisticated use cases. The addsvc example shows how Failer
|
||||
// should be used by a complete application.
|
||||
type Failer interface {
|
||||
Failed() error
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
# package metrics
|
||||
|
||||
`package metrics` provides a set of uniform interfaces for service instrumentation.
|
||||
It has
|
||||
[counters](http://prometheus.io/docs/concepts/metric_types/#counter),
|
||||
[gauges](http://prometheus.io/docs/concepts/metric_types/#gauge), and
|
||||
[histograms](http://prometheus.io/docs/concepts/metric_types/#histogram),
|
||||
and provides adapters to popular metrics packages, like
|
||||
[expvar](https://golang.org/pkg/expvar),
|
||||
[StatsD](https://github.com/etsy/statsd), and
|
||||
[Prometheus](https://prometheus.io).
|
||||
|
||||
## Rationale
|
||||
|
||||
Code instrumentation is absolutely essential to achieve
|
||||
[observability](https://speakerdeck.com/mattheath/observability-in-micro-service-architectures)
|
||||
into a distributed system.
|
||||
Metrics and instrumentation tools have coalesced around a few well-defined idioms.
|
||||
`package metrics` provides a common, minimal interface those idioms for service authors.
|
||||
|
||||
## Usage
|
||||
|
||||
A simple counter, exported via expvar.
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/expvar"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var myCount metrics.Counter
|
||||
myCount = expvar.NewCounter("my_count")
|
||||
myCount.Add(1)
|
||||
}
|
||||
```
|
||||
|
||||
A histogram for request duration,
|
||||
exported via a Prometheus summary with dynamically-computed quantiles.
|
||||
|
||||
```go
|
||||
import (
|
||||
"time"
|
||||
|
||||
stdprometheus "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/prometheus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var dur metrics.Histogram = prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
|
||||
Namespace: "myservice",
|
||||
Subsystem: "api",
|
||||
Name: "request_duration_seconds",
|
||||
Help: "Total time spent serving requests.",
|
||||
}, []string{})
|
||||
// ...
|
||||
}
|
||||
|
||||
func handleRequest(dur metrics.Histogram) {
|
||||
defer func(begin time.Time) { dur.Observe(time.Since(begin).Seconds()) }(time.Now())
|
||||
// handle request
|
||||
}
|
||||
```
|
||||
|
||||
A gauge for the number of goroutines currently running, exported via StatsD.
|
||||
|
||||
```go
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/statsd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
statsd := statsd.New("foo_svc.", log.NewNopLogger())
|
||||
report := time.NewTicker(5 * time.Second)
|
||||
defer report.Stop()
|
||||
go statsd.SendLoop(context.Background(), report.C, "tcp", "statsd.internal:8125")
|
||||
goroutines := statsd.NewGauge("goroutine_count")
|
||||
go exportGoroutines(goroutines)
|
||||
// ...
|
||||
}
|
||||
|
||||
func exportGoroutines(g metrics.Gauge) {
|
||||
for range time.Tick(time.Second) {
|
||||
g.Set(float64(runtime.NumGoroutine()))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For more information, see [the package documentation](https://godoc.org/github.com/go-kit/kit/metrics).
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
// Package metrics provides a framework for application instrumentation. It's
|
||||
// primarily designed to help you get started with good and robust
|
||||
// instrumentation, and to help you migrate from a less-capable system like
|
||||
// Graphite to a more-capable system like Prometheus. If your organization has
|
||||
// already standardized on an instrumentation system like Prometheus, and has no
|
||||
// plans to change, it may make sense to use that system's instrumentation
|
||||
// library directly.
|
||||
//
|
||||
// This package provides three core metric abstractions (Counter, Gauge, and
|
||||
// Histogram) and implementations for almost all common instrumentation
|
||||
// backends. Each metric has an observation method (Add, Set, or Observe,
|
||||
// respectively) used to record values, and a With method to "scope" the
|
||||
// observation by various parameters. For example, you might have a Histogram to
|
||||
// record request durations, parameterized by the method that's being called.
|
||||
//
|
||||
// var requestDuration metrics.Histogram
|
||||
// // ...
|
||||
// requestDuration.With("method", "MyMethod").Observe(time.Since(begin))
|
||||
//
|
||||
// This allows a single high-level metrics object (requestDuration) to work with
|
||||
// many code paths somewhat dynamically. The concept of With is fully supported
|
||||
// in some backends like Prometheus, and not supported in other backends like
|
||||
// Graphite. So, With may be a no-op, depending on the concrete implementation
|
||||
// you choose. Please check the implementation to know for sure. For
|
||||
// implementations that don't provide With, it's necessary to fully parameterize
|
||||
// each metric in the metric name, e.g.
|
||||
//
|
||||
// // Statsd
|
||||
// c := statsd.NewCounter("request_duration_MyMethod_200")
|
||||
// c.Add(1)
|
||||
//
|
||||
// // Prometheus
|
||||
// c := prometheus.NewCounter(stdprometheus.CounterOpts{
|
||||
// Name: "request_duration",
|
||||
// ...
|
||||
// }, []string{"method", "status_code"})
|
||||
// c.With("method", "MyMethod", "status_code", strconv.Itoa(code)).Add(1)
|
||||
//
|
||||
// Usage
|
||||
//
|
||||
// Metrics are dependencies, and should be passed to the components that need
|
||||
// them in the same way you'd construct and pass a database handle, or reference
|
||||
// to another component. Metrics should *not* be created in the global scope.
|
||||
// Instead, instantiate metrics in your func main, using whichever concrete
|
||||
// implementation is appropriate for your organization.
|
||||
//
|
||||
// latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
|
||||
// Namespace: "myteam",
|
||||
// Subsystem: "foosvc",
|
||||
// Name: "request_latency_seconds",
|
||||
// Help: "Incoming request latency in seconds.",
|
||||
// }, []string{"method", "status_code"})
|
||||
//
|
||||
// Write your components to take the metrics they will use as parameters to
|
||||
// their constructors. Use the interface types, not the concrete types. That is,
|
||||
//
|
||||
// // NewAPI takes metrics.Histogram, not *prometheus.Summary
|
||||
// func NewAPI(s Store, logger log.Logger, latency metrics.Histogram) *API {
|
||||
// // ...
|
||||
// }
|
||||
//
|
||||
// func (a *API) ServeFoo(w http.ResponseWriter, r *http.Request) {
|
||||
// begin := time.Now()
|
||||
// // ...
|
||||
// a.latency.Observe(time.Since(begin).Seconds())
|
||||
// }
|
||||
//
|
||||
// Finally, pass the metrics as dependencies when building your object graph.
|
||||
// This should happen in func main, not in the global scope.
|
||||
//
|
||||
// api := NewAPI(store, logger, latency)
|
||||
// http.ListenAndServe("/", api)
|
||||
//
|
||||
// Note that metrics are "write-only" interfaces.
|
||||
//
|
||||
// Implementation details
|
||||
//
|
||||
// All metrics are safe for concurrent use. Considerable design influence has
|
||||
// been taken from https://github.com/codahale/metrics and
|
||||
// https://prometheus.io.
|
||||
//
|
||||
// Each telemetry system has different semantics for label values, push vs.
|
||||
// pull, support for histograms, etc. These properties influence the design of
|
||||
// their respective packages. This table attempts to summarize the key points of
|
||||
// distinction.
|
||||
//
|
||||
// SYSTEM DIM COUNTERS GAUGES HISTOGRAMS
|
||||
// dogstatsd n batch, push-aggregate batch, push-aggregate native, batch, push-each
|
||||
// statsd 1 batch, push-aggregate batch, push-aggregate native, batch, push-each
|
||||
// graphite 1 batch, push-aggregate batch, push-aggregate synthetic, batch, push-aggregate
|
||||
// expvar 1 atomic atomic synthetic, batch, in-place expose
|
||||
// influx n custom custom custom
|
||||
// prometheus n native native native
|
||||
// pcp 1 native native native
|
||||
// cloudwatch n batch push-aggregate batch push-aggregate synthetic, batch, push-aggregate
|
||||
//
|
||||
package metrics
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package lv
|
||||
|
||||
// LabelValues is a type alias that provides validation on its With method.
|
||||
// Metrics may include it as a member to help them satisfy With semantics and
|
||||
// save some code duplication.
|
||||
type LabelValues []string
|
||||
|
||||
// With validates the input, and returns a new aggregate labelValues.
|
||||
func (lvs LabelValues) With(labelValues ...string) LabelValues {
|
||||
if len(labelValues)%2 != 0 {
|
||||
labelValues = append(labelValues, "unknown")
|
||||
}
|
||||
return append(lvs, labelValues...)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user