deployment

.gitignore

@@ -1,3 +1,6 @@
 /node_modules
 /indonesia_stocks
-/onnxruntime
+/onnxruntime
+/onnxruntime-linux-x64-1.21.0
+/models
+getmodels.sh

Dockerfile

@@ -1,26 +1,44 @@
 FROM golang:1.21.13-alpine3.20 
 
 LABEL creator="al-fariqy raihan"
+LABEL npm="202143501514"
 
 ENV APP_DIR=/thesis_forecasting_website \
     GO111MODULE=on \
-    CGO_ENABLED=0
+    CGO_ENABLED=0 
 
 WORKDIR ${APP_DIR}
 
-RUN apk add --no-cache git git-lfs curl \
+RUN apk add --no-cache git git-lfs curl tzdata \
     && git lfs install
+ENV TZ=Asia/Jakarta
 
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
 
+# NOTES:
+# gw pake Hugging Face (Docker), 
+# gak bisa multi-stage build bjirrr... wkwk..
 RUN go build -o main . \
     && go clean -modcache \
     && rm -rf /root/.cache/go-build /root/go/pkg
 
-RUN git clone https://huggingface.co/datasets/qywok/indonesia_stocks
+RUN wget https://github.com/microsoft/onnxruntime/releases/download/v1.21.0/onnxruntime-linux-x64-1.21.0.tgz && \
+    tar -xvzf onnxruntime-linux-x64-1.21.0.tgz && \
+    rm -rf onnxruntime-linux-x64-1.21.0.tgz && \
+    mv ./onnxruntime-linux-x64-1.21.0 ./onnxruntime
+
+RUN git lfs install && \
+    git clone https://huggingface.co/datasets/qywok/indonesia_stocks && \
+    mkdir -p models && \
+    for i in $(seq 1 10); do \
+      git clone https://huggingface.co/qywok/stock_models_$i && \
+      cd stock_models_$i && git lfs pull && cd .. && \
+      mv stock_models_$i/*.onnx models/ && \
+      rm -rf stock_models_$i; \
+    done
 
 RUN chmod -R 755 ${APP_DIR}
 

handlers/inference_handler.go

@@ -0,0 +1 @@
+package handlers

inferences/bigru_inference.go

@@ -1 +0,0 @@
-package inferences

inferences/bilstm_inference.go

@@ -1 +0,0 @@
-package inferences

inferences/stock_prediction.go

@@ -0,0 +1,161 @@
+package inferences
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+
+	loaders     "thesis_forecasting_website/loaders"
+	onnxruntime "github.com/belajarqywok/onnxruntime_go"
+	"sync"
+)
+
+var once sync.Once
+var isInit bool
+
+func denormalization(data, minValue, maxValue float32) float32 {
+	return (data * (maxValue - minValue)) + minValue
+}
+
+type StockRequest struct {
+	Issuer string `json:"issuer"`
+	Days   int    `json:"days"`
+}
+
+type StockPrice struct {
+	Date  string  `json:"date"`
+	Price float64 `json:"price"`
+}
+
+type StockResponse struct {
+	Actuals    []StockPrice `json:"actuals"`
+	Prediction []StockPrice `json:"prediction"`
+}
+
+func StockPredictionHandler(c *fiber.Ctx) error {
+	req := new(StockRequest)
+	if err := c.BodyParser(req); err != nil {
+		return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+			"error": "Invalid request body",
+		})
+	}
+
+	if req.Days <= 0 { req.Days = 7 }
+
+	once.Do(func() {
+		onnxruntime.SetSharedLibraryPath("./onnxruntime-linux-x64-1.21.0/lib/libonnxruntime.so")
+		if err := onnxruntime.InitializeEnvironment(); err != nil {
+			log.Fatal("Error initializing ONNX runtime: ", err)
+		}
+
+		isInit = true
+	})
+
+	if !isInit { log.Fatal("ONNX runtime not initialized") }
+
+	// load dataset
+	dataset_csv_path := fmt.Sprintf("./indonesia_stocks/modeling_datas/%s.csv", req.Issuer)
+	data, err := loaders.DatasetLoader(dataset_csv_path)
+	if err != nil {
+		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+			"error": "Error loading CSV",
+		})
+	}
+
+	// load scaler
+	minmax_json_path := fmt.Sprintf("./indonesia_stocks/min_max/%s.json", req.Issuer)
+	scalers, err := loaders.ScalersLoader(minmax_json_path)
+	if err != nil {
+		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+			"error": "Error loading scalers",
+		})
+	}
+
+	n := 30
+	if len(data) < n { n = len(data) }
+	lastDataActual := data[len(data) - n:]
+
+	var actuals []StockPrice
+	for _, row := range lastDataActual {
+		date := row[0].(string)
+		closeVal := row[1].(float32)
+
+		closePrice := denormalization(
+			closeVal,
+			scalers.MinValue["Close"],
+			scalers.MaxValue["Close"],
+		)
+
+		actuals = append(actuals, StockPrice{
+			Date:  date,
+			Price: float64(closePrice),
+		})
+	}
+
+
+	// Prepare input for model
+	sequenceLength := int64(60)
+	featureSize    := int64(5)
+	lastData       := data[len(data) - int(sequenceLength):]
+
+	inputData  := make([]float32, sequenceLength * featureSize)
+	for i, row := range lastData {
+		for j := 1; j <= int(featureSize); j++ {
+			val, ok := row[j].(float32)
+			if !ok {
+				log.Fatalf("Expected float32 at row %d col %d, got %T", i, j, row[j])
+			}
+
+			inputData[i*int(featureSize) + (j-1)] = val
+		}
+	}
+
+	inputShape := onnxruntime.NewShape(1, sequenceLength, featureSize)
+	inputTensor, err := onnxruntime.NewTensor(inputShape, inputData)
+	if err != nil { log.Fatalf("Error creating input tensor: %v", err) }
+
+	outputShape := onnxruntime.NewShape(1, 1)
+	outputTensor, err := onnxruntime.NewEmptyTensor[float32](outputShape)
+	if err != nil { log.Fatalf("Error creating output tensor: %v", err) }
+
+	model_onnx_path := fmt.Sprintf("./models/%s.onnx", req.Issuer)
+	session, err := onnxruntime.NewAdvancedSession(
+		model_onnx_path,
+		[]string{"input"}, []string{"output"},
+		[]onnxruntime.ArbitraryTensor{inputTensor},
+		[]onnxruntime.ArbitraryTensor{outputTensor}, nil,
+	)
+
+	if err != nil { log.Fatalf("Error initializing ONNX session: %v", err) }
+
+	// generate predictions
+	var predicted []StockPrice
+	lastDate, _ := time.Parse("2006-01-02", actuals[len(actuals)-1].Date)
+
+	for i := 0; i < req.Days; i++ {
+		if err := session.Run(); err != nil {
+			log.Fatalf("Error running model: %v", err)
+		}
+
+		predictedClose := outputTensor.GetData()[0]
+		denormPrice := denormalization(predictedClose, scalers.MinValue["Close"], scalers.MaxValue["Close"])
+
+		lastDate = lastDate.AddDate(0, 0, 1)
+		predicted = append(predicted, StockPrice{
+			Date:  lastDate.Format("2006-01-02"),
+			Price: float64(denormPrice),
+		})
+
+		copy(inputData, inputData[int(featureSize):])
+		inputData[len(inputData)-1] = predictedClose
+	}
+
+	resp := StockResponse{
+		Actuals:    actuals,
+		Prediction: predicted,
+	}
+
+	return c.JSON(resp)
+}

loaders/fundamental_loader.go

@@ -5,6 +5,43 @@ import (
 	"encoding/json"
 )
 
+type Fundamental struct {
+	FontawesomeIcon      string `json:"fontawesome_icon"`
+	Symbol               string `json:"symbol"`
+	SectorID             string `json:"sector_id"`
+	ShortName            string `json:"shortName"`
+
+  Address              string `json:"address"`
+  Phone                string `json:"phone"`
+  Website              string `json:"website"`
+  MarketCap            string `json:"marketCap"`
+  DividendRate         string `json:"dividendRate"`
+  DividendYield        string `json:"dividendYield"`
+  EarningsGrowth       string `json:"earningsGrowth"`
+  ProfitMargins        string `json:"profitMargins"`
+  GrossMargins         string `json:"grossMargins"`
+  Beta                 string `json:"beta"`
+  BookValue            string `json:"bookValue"`
+  PriceToBook          string `json:"priceToBook"`
+
+  QuickRatio           string `json:"quickRatio"`
+  CurrentRatio         string `json:"currentRatio"`
+  DebtToEquity         string `json:"debtToEquity"`
+  RevenuePerShare      string `json:"revenuePerShare"`
+  RevenueGrowth        string `json:"revenueGrowth"`
+  Ebitda               string `json:"ebitda"`
+  RegularMarketChange  string `json:"regularMarketChange"`
+  PayoutRatio          string `json:"payoutRatio"`
+  TrailingPE           string `json:"trailingPE"`
+  ForwardPE            string `json:"forwardPE"`
+  TrailingEps          string `json:"trailingEps"`
+  ForwardEps           string `json:"forwardEps"`
+}
+
+type FundamentalsWrapper struct {
+	Fundamentals Fundamental `json:"fundamentals"`
+}
+
 func FundamentalLoader(path string) (*Fundamental, error) {
 	file, err := os.ReadFile(path)
 	if err != nil {

loaders/fundamental_struct.go

@@ -1,38 +0,0 @@
-package loaders
-
-type Fundamental struct {
-	FontawesomeIcon      string `json:"fontawesome_icon"`
-	Symbol               string `json:"symbol"`
-	SectorID             string `json:"sector_id"`
-	ShortName            string `json:"shortName"`
-
-  Address              string `json:"address"`
-  Phone                string `json:"phone"`
-  Website              string `json:"website"`
-  MarketCap            string `json:"marketCap"`
-  DividendRate         string `json:"dividendRate"`
-  DividendYield        string `json:"dividendYield"`
-  EarningsGrowth       string `json:"earningsGrowth"`
-  ProfitMargins        string `json:"profitMargins"`
-  GrossMargins         string `json:"grossMargins"`
-  Beta                 string `json:"beta"`
-  BookValue            string `json:"bookValue"`
-  PriceToBook          string `json:"priceToBook"`
-
-  QuickRatio           string `json:"quickRatio"`
-  CurrentRatio         string `json:"currentRatio"`
-  DebtToEquity         string `json:"debtToEquity"`
-  RevenuePerShare      string `json:"revenuePerShare"`
-  RevenueGrowth        string `json:"revenueGrowth"`
-  Ebitda               string `json:"ebitda"`
-  RegularMarketChange  string `json:"regularMarketChange"`
-  PayoutRatio          string `json:"payoutRatio"`
-  TrailingPE           string `json:"trailingPE"`
-  ForwardPE            string `json:"forwardPE"`
-  TrailingEps          string `json:"trailingEps"`
-  ForwardEps           string `json:"forwardEps"`
-}
-
-type FundamentalsWrapper struct {
-	Fundamentals Fundamental `json:"fundamentals"`
-}

loaders/historical_loader.go

@@ -5,6 +5,20 @@ import (
 	"encoding/json"
 )
 
+type Historical struct {
+	Date           string   `json:"date"`
+	FullDate       string   `json:"full_date"`
+	Open          float64   `json:"open"`
+	High          float64   `json:"high"`
+	Low           float64   `json:"low"`
+	Close         float64   `json:"close"`
+  Volume        float64   `json:"volume"`
+}
+
+type HistoricalsWrapper struct {
+	Historicals   []Historical   `json:"historicals"`
+}
+
 func HistoricalLoader(path string) ([]Historical, error) {
 	file, err := os.ReadFile(path)
 	if err != nil {

loaders/historical_struct.go

@@ -1,15 +0,0 @@
-package loaders
-
-type Historical struct {
-	Date           string   `json:"date"`
-	FullDate       string   `json:"full_date"`
-	Open          float64   `json:"open"`
-	High          float64   `json:"high"`
-	Low           float64   `json:"low"`
-	Close         float64   `json:"close"`
-  Volume        float64   `json:"volume"`
-}
-
-type HistoricalsWrapper struct {
-	Historicals   []Historical   `json:"historicals"`
-}

loaders/indicator_loader.go

@@ -5,6 +5,18 @@ import (
 	"encoding/json"
 )
 
+type Indicator struct {
+	Date            string  `json:"date"`
+	FullDate        string  `json:"full_date"`
+	MFI            float64  `json:"MFI"`
+	RSI            float64  `json:"RSI"`
+	MACD           float64  `json:"MACD"`
+}
+
+type IndicatorsWrapper struct {
+	Indicators   []Indicator   `json:"indicators"`
+}
+
 func IndicatorLoader(path string) ([]Indicator, error) {
 	file, err := os.ReadFile(path)
 	if err != nil {

loaders/indicator_struct.go

@@ -1,13 +0,0 @@
-package loaders
-
-type Indicator struct {
-	Date            string  `json:"date"`
-	FullDate        string  `json:"full_date"`
-	MFI            float64  `json:"MFI"`
-	RSI            float64  `json:"RSI"`
-	MACD           float64  `json:"MACD"`
-}
-
-type IndicatorsWrapper struct {
-	Indicators   []Indicator   `json:"indicators"`
-}

loaders/inference_loader.go

@@ -0,0 +1,64 @@
+package loaders
+
+import (
+	"os"
+	"fmt"
+	"strconv"
+	"io/ioutil"
+	"encoding/csv"
+	"encoding/json"
+)
+
+type Scalers struct {
+	MinValue map[string]float32 `json:"min_value"`
+	MaxValue map[string]float32 `json:"max_value"`
+}
+
+func ScalersLoader(filename string) (Scalers, error) {
+	var scalers_data Scalers
+
+	scalers_json, err := ioutil.ReadFile(filename)
+	if err != nil { return scalers_data, err }
+
+	err = json.Unmarshal(scalers_json, &scalers_data)
+	if err != nil { return scalers_data, err }
+
+	return scalers_data, nil
+}
+
+func DatasetLoader(filepath string) ([][]interface{}, error) {
+	csv_file, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer csv_file.Close()
+
+	csv_reader := csv.NewReader(csv_file)
+	csv_records, err := csv_reader.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	var csv_data [][]interface{}
+	for _, csv_row := range csv_records[1:] {
+		var row_data []interface{}
+
+		// kolom pertama tetap string (Date)
+		row_data = append(row_data, csv_row[0])
+
+		// sisanya float32
+		for _, csv_row_val := range csv_row[1:] {
+			val, err := strconv.ParseFloat(csv_row_val, 32)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing value '%s': %v", csv_row_val, err)
+			}
+			row_data = append(row_data, float32(val))
+		}
+
+		csv_data = append(csv_data, row_data)
+	}
+
+	return csv_data, nil
+}
+
+

loaders/issuer_loader.go

@@ -5,6 +5,19 @@ import (
 	"encoding/json"
 )
 
+type Issuer struct {
+	FontawesomeIcon string `json:"fontawesome_icon"`
+	Symbol          string `json:"symbol"`
+	SectorID        string `json:"sector_id"`
+	ShortName       string `json:"shortName"`
+	Beta            string `json:"beta"`
+	DividendYield   string `json:"dividendYield"`
+}
+
+type IssuersWrapper struct {
+	Issuers []Issuer `json:"infographics"`
+}
+
 func IssuerLoader(path string) ([]Issuer, error) {
 	file, err := os.ReadFile(path)
 	if err != nil {

loaders/issuer_struct.go

@@ -1,14 +0,0 @@
-package loaders
-
-type Issuer struct {
-	FontawesomeIcon string `json:"fontawesome_icon"`
-	Symbol          string `json:"symbol"`
-	SectorID        string `json:"sector_id"`
-	ShortName       string `json:"shortName"`
-	Beta            string `json:"beta"`
-	DividendYield   string `json:"dividendYield"`
-}
-
-type IssuersWrapper struct {
-	Issuers []Issuer `json:"infographics"`
-}

loaders/sector_loader.go

@@ -5,11 +5,13 @@ import (
 	"encoding/json"
 )
 
+type SectorsWrapper struct {
+	Sectors []string `json:"sectors"`
+}
+
 func SectorLoader(path string) ([]string, error) {
 	file, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
+	if err != nil { return nil, err }
 
 	var data SectorsWrapper
 	if err := json.Unmarshal(file, &data); err != nil {

loaders/sector_struct.go

@@ -1,5 +0,0 @@
-package loaders
-
-type SectorsWrapper struct {
-	Sectors []string `json:"sectors"`
-}

main.go

@@ -15,6 +15,7 @@ import (
 
   helpers      "thesis_forecasting_website/helpers"
   handlers     "thesis_forecasting_website/handlers"
+  inferences   "thesis_forecasting_website/inferences"
   middlewares  "thesis_forecasting_website/middlewares"
 )
 
@@ -36,6 +37,7 @@ func main() {
 
   forecasting_service.Get("/", handlers.IssuerHandler)
   forecasting_service.Get("/infographic", handlers.InfographicHandler)
+  forecasting_service.Post("/prediction", inferences.StockPredictionHandler)
 
   host := os.Getenv("FORECASTING_SERVICE_HOST")
 	port := os.Getenv("FORECASTING_SERVICE_PORT")

makefile

@@ -10,5 +10,14 @@ run:
 	npm run min:js:infographic:stock_historical:table
 	go run main.go
 
+getmodels:
+	apt install -y git git-lfs
+	git lfs install
 
-	
+	mkdir -p models
+	for i in $(seq 1 10); do
+		git clone https://huggingface.co/qywok/stock_models_$i
+		cd stock_models_$i && git lfs pull && cd ..
+		mv stock_models_$i/*.onnx models/
+		rm -rf stock_models_$i
+	done