-𝐫𝐮𝐬𝐭𝐥𝐞𝐫 is a web scraping service that scrapes several stock market providers for stock pricing data. It is built using the Rust programming language.
+𝐫𝐮𝐬𝐭𝐥𝐞𝐫 ⫮ 𝐜𝐨𝐫𝐞 is a library that contains the core functionality for `rustler`, a web scraping service that scrapes several stock market providers for stock pricing data. It is built using the Rust programming language.
-## Framweork
+## Why "rustler"
-- [Tonic (grpc)](https://docs.rs/tonic/latest/tonic/index.html)
-- websockets:
- - [tokio-tungstenite](https://docs.rs/tokio-tungstenite/latest/tokio_tungstenite/)
- - [fastwebsockets](https://crates.io/crates/fastwebsockets)
- - [embedded-websocket](https://crates.io/crates/embedded-websocket) - bajo nivel - small
- - [web-socket](https://crates.io/crates/web-socket) - supuestamente el mas rapido
-- Rx Rust:
- - [rxrust](https://crates.io/crates/rxrust)
- - [another-rxrust](https://crates.io/crates/another-rxrust) parece un poco mejor
+A `rustler` is a person who steals live***stock***. Well, this library is a service that collects _stock_ market data from the internet. So, it's a "_rustler_" for stock market data.
-## Commands
+Also, this library is built using the `Rust` programming language... so, ***rust***ler 😊
-```bash
-# add a dependency to a project
-cargo add {dependency} -p {project}
+## What this library includes
-# example
-cargo add tokio-tungstenite -p gateway
-```
\ No newline at end of file
+This library defines the core functionality for a `rustler`. It includes the following:
+
+- A [`Rustler`](./lib/rustlers/rustlers.rs) trait that defines the core functionality for a `rustler`.
+- A [`RustlersSvc`](./lib/rustlers/svc.rs) which orchestrates the `rustlers` at runtime, scheduling them to scrape stock pricing data between market hours.
+
+Apart from the above, this library also defines:
+
+- a [database schema](./lib/entities/orm/) for storing market hours, which is used by the `RustlersSvc` to schedule the `rustlers`.
+- initial [database migrations](./lib/entities/migration) to create the schema.
+- a [grpc server](./lib/grpc/) to interact with the rustlers database.
+- a [websocket gateway server](./lib/socket/) to stream stock pricing data to subscribed clients
+
+> [!NOTE]
+>
+> This library defines a _rustler_ as a service that scrapes stock pricing data for a
+> particular market.
+>
+> Although this library contains the core and abstract functionality for the rustlers, it doesn't include any concrete implementation for them.
+>
+> Actual concrete implementations for each market cannot be published for many reasons.
+
+## Example
+
+Check the [examples](./examples) directory for an example of how to use this library.
diff --git a/Taskfile.yaml b/Taskfile.yaml
index 26987e0..2a9109c 100644
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@@ -16,7 +16,7 @@ tasks:
run:watch:
desc: 🚀 watch rustler
cmds:
- - cargo watch -c -x "run"
+ - cargo watch -c -x "run --example=rustler"
build:watch:
desc: 🚀 watch rustler «build»
diff --git a/check_size.py b/check_size.py
deleted file mode 100644
index f88314a..0000000
--- a/check_size.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-small script to run after build, to check if there was a significant
-change on executable size, compared to the previous build.
-
-this aims to detect unwanted big differences before it's too late
-"""
-
-
-import os
-import pathlib
-
-curr_dir = pathlib.Path(os.getcwd())
-sizefile_path = pathlib.Path(curr_dir.joinpath('.task'))
-
-
-def bad(txt):
- return '\033[91m' + txt + '\033[0m'
-
-
-def good(txt):
- return '\033[92m' + txt + '\033[0m'
-
-
-def head(txt):
- return '\033[94m' + txt + '\033[0m'
-
-
-files = {
- 'release': curr_dir.joinpath('target/release/rustler.exe'),
- 'debug': curr_dir.joinpath('target/debug/rustler.exe'),
-}
-
-print("\n🧉 » exe file sizes change\n")
-
-for key, exe in files.items():
- if exe.is_file():
- sizefile = sizefile_path.joinpath(key)
- new_size: float = os.stat(exe).st_size / 1024
- old_size: float
-
- try:
- with open(sizefile, 'r') as f:
- old_size = float(f.read())
- except FileNotFoundError:
- old_size = 0
-
- # diff: str = f'{old_size:.0f}kb'
- diff = new_size - old_size
- diff_str = f"{'+' if diff > 0 else '=' if diff == 0 else ''}{diff:.0f}kb"
-
- fmt = bad if diff > 10 else good
-
- sizefile.parent.mkdir(parents=True, exist_ok=True)
-
- print(
- f'{head(key)}: {fmt(diff_str)} (prev: {old_size}kb, now: {fmt(f"{str(new_size)}kb")})'
- )
-
- with open(sizefile, 'w') as f:
- f.write(f'{new_size}')
diff --git a/entities/Cargo.toml b/entities/Cargo.toml
deleted file mode 100644
index a3ed977..0000000
--- a/entities/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "entities"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "entities"
-path = "src/lib.rs"
-
-[dependencies]
-# common
-eyre = { workspace = true, default-features = false }
-lool = { workspace = true, features = [ "cli", "cli.stylize", "macros", "logger" ] }
-
-# external
-sea-orm = { version = "0.12.15", features = [
- "runtime-tokio-native-tls",
- "sqlx-sqlite",
- "macros",
-] }
diff --git a/rustlers/src/rustlers/binance.rs b/examples/binance/mod.rs
similarity index 67%
rename from rustlers/src/rustlers/binance.rs
rename to examples/binance/mod.rs
index a34d249..ede5f57 100644
--- a/rustlers/src/rustlers/binance.rs
+++ b/examples/binance/mod.rs
@@ -1,33 +1,27 @@
use {
- crate::{
+ async_trait::async_trait,
+ eyre::Result,
+ lool::logger::info,
+ rustler_core::{
rustler,
rustlers::{Rustler, RustlerAccessor, RustlerStatus, Ticker},
},
- async_trait::async_trait,
- eyre::Result,
- lool::{cli::stylize::Stylize, logger::info},
std::collections::HashMap,
};
-const BINANCE_WSS_URL: &str = "wss://stream.binance.com:9443/stream";
-// const MANUAL_CLOSE_CODE: u16 = 4663;
-// const MANUAL_CLOSE_REASON: &str = "Manually Disconnected";
-
rustler!(
- /// 🤠 » **binance rustler**
- ///
- /// A rustler that steals quotes from Binance
- pub struct BinanceRustler {}
+ /// A fake rustler that does nothing but changing between different statuses.
+ pub struct FooRustler {}
);
-impl BinanceRustler {
+impl FooRustler {
pub fn create() -> impl Rustler {
Self::default()
}
}
#[async_trait]
-impl Rustler for BinanceRustler {
+impl Rustler for FooRustler {
async fn connect(&mut self) -> Result<()> {
if self.status == RustlerStatus::Connected || self.status == RustlerStatus::Connecting {
return Ok(());
@@ -35,10 +29,7 @@ impl Rustler for BinanceRustler {
self.set_status(RustlerStatus::Connecting)?;
- info!(
- "Connecting to Binance WSS: {}",
- BINANCE_WSS_URL.bright_green()
- );
+ info!("Connecting to data source");
self.set_status(RustlerStatus::Connected)?;
@@ -53,7 +44,7 @@ impl Rustler for BinanceRustler {
self.set_status(RustlerStatus::Disconnecting)?;
- info!("Disconnecting from Binance WSS");
+ info!("Disconnecting from data source");
self.set_status(RustlerStatus::Disconnected)?;
diff --git a/app/main.rs b/examples/rustler.rs
similarity index 58%
rename from app/main.rs
rename to examples/rustler.rs
index d852476..ea40fb0 100644
--- a/app/main.rs
+++ b/examples/rustler.rs
@@ -1,24 +1,25 @@
+mod binance;
+
use {
+ binance::FooRustler,
dotenvy::dotenv,
- eyre::Result,
+ eyre::{set_hook, DefaultHandler, Result},
lool::logger::{info, ConsoleLogger, Level},
- rustlers::{rustlerjar, rustlers::binance::BinanceRustler, svc::RustlersSvc},
+ rustler_core::{entities::db::get_connection, grpc, rustlerjar, rustlers::svc::RustlersSvc},
tokio::join,
};
-// TODO: here we will trigger the start of both the grpc server and the websocket gateway
-// look at: https://github.com/hyperium/tonic/discussions/740
-
#[tokio::main]
async fn main() -> Result<()> {
+ set_hook(Box::new(DefaultHandler::default_with))?;
ConsoleLogger::default_setup(Level::Trace, "rustler")?;
dotenv()?;
- let conn = entities::db::get_connection().await?;
+ let conn = get_connection().await?;
let mut rustler = RustlersSvc::new(
conn.clone(),
rustlerjar! {
- "BINANCE" => BinanceRustler
+ "BINANCE" => FooRustler
},
)
.await;
diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml
deleted file mode 100644
index 04e8d20..0000000
--- a/gateway/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "gateway"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "gateway"
-path = "src/lib.rs"
-
-[dependencies]
-entities = { path = "../entities" }
-eyre = { workspace = true, default-features = false }
-
diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs
deleted file mode 100644
index e69de29..0000000
diff --git a/grpc/Cargo.toml b/grpc/Cargo.toml
deleted file mode 100644
index acd802c..0000000
--- a/grpc/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "grpc"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "grpc"
-path = "src/lib.rs"
-
-[dependencies]
-# common
-lool = { workspace = true, features = ["logger"]}
-eyre = { workspace = true, default-features = false }
-
-# internal
-entities = { path = "../entities" }
-
-# external
-prost = "0.12.3"
-tonic = "0.11.0"
-
-[build-dependencies]
-tonic-build = "0.11.0"
-
diff --git a/grpc/build.rs b/lib/build.rs
similarity index 71%
rename from grpc/build.rs
rename to lib/build.rs
index 2d85665..1ac75f0 100644
--- a/grpc/build.rs
+++ b/lib/build.rs
@@ -1,5 +1,9 @@
fn main() {
- let proto_files = vec!["./proto/rustler.proto", "./proto/market.proto", "./proto/ticker.proto"];
+ let proto_files = vec![
+ "./lib/grpc/proto/rustler.proto",
+ "./lib/grpc/proto/market.proto",
+ "./lib/grpc/proto/ticker.proto",
+ ];
for proto_file in proto_files {
compile_proto(proto_file);
diff --git a/migration/src/m20220101_000001_create_table_market.rs b/lib/entities/migration/m20220101_000001_create_table_market.rs
similarity index 98%
rename from migration/src/m20220101_000001_create_table_market.rs
rename to lib/entities/migration/m20220101_000001_create_table_market.rs
index 45c2c2f..3af09fb 100644
--- a/migration/src/m20220101_000001_create_table_market.rs
+++ b/lib/entities/migration/m20220101_000001_create_table_market.rs
@@ -1,6 +1,7 @@
use sea_orm_migration::{async_trait::async_trait, prelude::*};
#[derive(DeriveMigrationName)]
+/// 🤠 » create table `market`
pub struct Migration;
#[async_trait]
diff --git a/migration/src/m20240325_200049_create_table_ticker.rs b/lib/entities/migration/m20240325_200049_create_table_ticker.rs
similarity index 97%
rename from migration/src/m20240325_200049_create_table_ticker.rs
rename to lib/entities/migration/m20240325_200049_create_table_ticker.rs
index d006c58..6fea9fa 100644
--- a/migration/src/m20240325_200049_create_table_ticker.rs
+++ b/lib/entities/migration/m20240325_200049_create_table_ticker.rs
@@ -1,6 +1,7 @@
use sea_orm_migration::{async_trait::async_trait, prelude::*};
#[derive(DeriveMigrationName)]
+/// 🤠 » create table `ticker`
pub struct Migration;
#[async_trait]
diff --git a/lib/entities/migration/mod.rs b/lib/entities/migration/mod.rs
new file mode 100644
index 0000000..9b9514c
--- /dev/null
+++ b/lib/entities/migration/mod.rs
@@ -0,0 +1,2 @@
+mod m20220101_000001_create_table_market;
+mod m20240325_200049_create_table_ticker;
diff --git a/entities/src/lib.rs b/lib/entities/mod.rs
similarity index 98%
rename from entities/src/lib.rs
rename to lib/entities/mod.rs
index d5ea9ea..7eaf2c5 100644
--- a/entities/src/lib.rs
+++ b/lib/entities/mod.rs
@@ -1,3 +1,4 @@
+pub mod migration;
pub use sea_orm;
mod orm {
diff --git a/entities/src/orm/market.rs b/lib/entities/orm/market.rs
similarity index 96%
rename from entities/src/orm/market.rs
rename to lib/entities/orm/market.rs
index 402f366..a04589d 100644
--- a/entities/src/orm/market.rs
+++ b/lib/entities/orm/market.rs
@@ -4,6 +4,7 @@ use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "market")]
+/// 🤠 » market entity model
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
diff --git a/entities/src/orm/ticker.rs b/lib/entities/orm/ticker.rs
similarity index 96%
rename from entities/src/orm/ticker.rs
rename to lib/entities/orm/ticker.rs
index 2e3e94c..92fda31 100644
--- a/entities/src/orm/ticker.rs
+++ b/lib/entities/orm/ticker.rs
@@ -4,6 +4,7 @@ use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "ticker")]
+/// 🤠 » ticker entity model
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
diff --git a/entities/src/services/market.rs b/lib/entities/services/market.rs
similarity index 83%
rename from entities/src/services/market.rs
rename to lib/entities/services/market.rs
index 72177b5..00acc94 100644
--- a/entities/src/services/market.rs
+++ b/lib/entities/services/market.rs
@@ -1,5 +1,5 @@
use {
- crate::{
+ crate::entities::{
market::{Entity as Market, Model as MarketModel},
ticker::{Entity as Ticker, Model as TickerModel},
},
@@ -7,6 +7,7 @@ use {
sea_orm::{DatabaseConnection, DbErr, EntityTrait, IntoActiveModel},
};
+/// 🤠 » service for the `Market` entity
pub struct Service {
conn: DatabaseConnection,
}
@@ -16,16 +17,19 @@ impl Service {
Self { conn }
}
+ /// 🤠 » gets all markets from the database
pub async fn get_all(&self) -> Result, DbErr> {
let markets = Market::find().all(&self.conn).await?;
Ok(markets)
}
+ /// 🤠 » gets a market by its id
pub async fn create(&self, market: MarketModel) -> Result {
Market::insert(market.clone().into_active_model()).exec(&self.conn).await?;
Ok(market)
}
+ /// 🤠 » gets all markets with their tickers
pub async fn get_all_with_tickers(
&self,
) -> Result)>, DbErr> {
diff --git a/entities/src/services/ticker.rs b/lib/entities/services/ticker.rs
similarity index 94%
rename from entities/src/services/ticker.rs
rename to lib/entities/services/ticker.rs
index 77c5494..f0c7ea0 100644
--- a/entities/src/services/ticker.rs
+++ b/lib/entities/services/ticker.rs
@@ -1,5 +1,5 @@
use {
- crate::{
+ crate::entities::{
orm::ticker,
ticker::{Entity as Ticker, Model as TickerModel},
},
@@ -7,6 +7,7 @@ use {
sea_orm::{ColumnTrait, DatabaseConnection, DbErr, EntityTrait, IntoActiveModel, QueryFilter},
};
+/// 🤠 » service for the `Ticker` entity
pub struct Service {
conn: DatabaseConnection,
}
diff --git a/grpc/src/lib.rs b/lib/grpc/mod.rs
similarity index 81%
rename from grpc/src/lib.rs
rename to lib/grpc/mod.rs
index 6d1d02f..5e7e19f 100644
--- a/grpc/src/lib.rs
+++ b/lib/grpc/mod.rs
@@ -1,12 +1,15 @@
mod services {
use {
- entities::sea_orm::{DbErr, SqlErr},
lool::s,
+ sea_orm::{DbErr, SqlErr},
};
+ /// market grpc services
pub mod market;
+ /// ticker grpc services
pub mod ticker;
+ /// general error handling for sql errors in grpc services
pub(crate) fn handle_sql_err(err: DbErr, action: &str, entity_name: &str) -> tonic::Status {
let sqlerr = err.sql_err();
diff --git a/grpc/proto/market.proto b/lib/grpc/proto/market.proto
similarity index 100%
rename from grpc/proto/market.proto
rename to lib/grpc/proto/market.proto
diff --git a/grpc/proto/rustler.proto b/lib/grpc/proto/rustler.proto
similarity index 100%
rename from grpc/proto/rustler.proto
rename to lib/grpc/proto/rustler.proto
diff --git a/grpc/proto/ticker.proto b/lib/grpc/proto/ticker.proto
similarity index 100%
rename from grpc/proto/ticker.proto
rename to lib/grpc/proto/ticker.proto
diff --git a/grpc/src/server.rs b/lib/grpc/server.rs
similarity index 79%
rename from grpc/src/server.rs
rename to lib/grpc/server.rs
index 9cb75c5..e002930 100644
--- a/grpc/src/server.rs
+++ b/lib/grpc/server.rs
@@ -1,15 +1,18 @@
use {
- crate::services,
- entities::{market, sea_orm::DatabaseConnection, ticker},
+ crate::{
+ entities::{market, ticker},
+ grpc::services,
+ },
eyre::Result,
lool::{cli::stylize::Stylize, logger::info},
+ sea_orm::DatabaseConnection,
std::net::SocketAddr,
tonic::transport::Server,
};
const RUSTLER_GRPC_API_ADDR: &str = "RUSTLER_GRPC_API_ADDR";
-/// Starts the gRPC server
+/// 🤠 » starts the rustler gRPC server
pub async fn start(conn: DatabaseConnection) -> Result<()> {
fn get_default_addr() -> String {
let addr = "0.0.0.0:50051";
@@ -36,8 +39,8 @@ pub async fn start(conn: DatabaseConnection) -> Result<()> {
);
Server::builder()
- .add_service(market_grpc.svc())
- .add_service(ticker_grpc.svc())
+ .add_service(market_grpc.svc()) // add the market api
+ .add_service(ticker_grpc.svc()) // add the ticker api
.serve(addr)
.await?;
diff --git a/grpc/src/services/market.rs b/lib/grpc/services/market.rs
similarity index 94%
rename from grpc/src/services/market.rs
rename to lib/grpc/services/market.rs
index 74068ca..39b5362 100644
--- a/grpc/src/services/market.rs
+++ b/lib/grpc/services/market.rs
@@ -1,7 +1,6 @@
use {
self::market_mod::Empty,
- crate::services::handle_sql_err,
- entities::market,
+ crate::{entities::market, grpc::services::handle_sql_err},
eyre::Result,
lool::logger::{error, info},
market_mod::{
@@ -48,6 +47,7 @@ impl Market {
}
}
+/// 🤠 » grpc Server to manage market entities
pub struct GrpcServer {
pub(crate) svc: market::Service,
}
@@ -59,6 +59,7 @@ impl GrpcServer {
}
}
+ /// 🤠 » creates the market api server
pub fn svc(self) -> MarketApiServer {
MarketApiServer::new(self)
}
diff --git a/grpc/src/services/ticker.rs b/lib/grpc/services/ticker.rs
similarity index 94%
rename from grpc/src/services/ticker.rs
rename to lib/grpc/services/ticker.rs
index cca1fb4..b32e6fc 100644
--- a/grpc/src/services/ticker.rs
+++ b/lib/grpc/services/ticker.rs
@@ -1,6 +1,5 @@
use {
- crate::services::handle_sql_err,
- entities::ticker,
+ crate::{entities::ticker, grpc::services::handle_sql_err},
eyre::Result,
lool::logger::{error, info},
std::{any::Any, fmt::Debug, time::Instant},
@@ -33,6 +32,7 @@ impl Ticker {
}
}
+/// 🤠 » grpc Server to manage ticker entities
pub struct GrpcServer {
pub(crate) svc: ticker::Service,
}
@@ -44,6 +44,7 @@ impl GrpcServer {
}
}
+ /// 🤠 » creates the ticker api server
pub fn svc(self) -> TickerApiServer {
TickerApiServer::new(self)
}
diff --git a/lib/lib.rs b/lib/lib.rs
new file mode 100644
index 0000000..e64b7ba
--- /dev/null
+++ b/lib/lib.rs
@@ -0,0 +1,4 @@
+pub mod entities;
+pub mod grpc;
+pub mod rustlers;
+pub mod socket;
diff --git a/lib/rustlers/mod.rs b/lib/rustlers/mod.rs
new file mode 100644
index 0000000..8e5f582
--- /dev/null
+++ b/lib/rustlers/mod.rs
@@ -0,0 +1,6 @@
+mod rustler;
+
+pub mod rustlerjar;
+pub mod svc;
+
+pub use rustler::*;
diff --git a/rustlers/src/rustlers.rs b/lib/rustlers/rustler.rs
similarity index 91%
rename from rustlers/src/rustlers.rs
rename to lib/rustlers/rustler.rs
index 44b0dce..aad90c3 100644
--- a/rustlers/src/rustlers.rs
+++ b/lib/rustlers/rustler.rs
@@ -1,11 +1,10 @@
-pub mod binance;
pub extern crate chrono;
pub extern crate eyre;
use {
+ crate::entities::{market, ticker},
async_trait::async_trait,
chrono::{DateTime, Local},
- entities::{market, ticker},
eyre::Result,
std::collections::HashMap,
};
@@ -59,6 +58,13 @@ pub struct ScrapperCallbacks {
pub on_message: Option Result<()>>,
}
+/// 🤠 » a scruct representing a ticker
+///
+/// in `rustler` a ticker is the union between a symbol (stock identifier) and its market
+///
+/// they `key` of a ticker is the concatenation of the market and the symbol separated by a colon
+///
+/// e.g. `AAPL` in the `NASDAQ` market would have the key `NASDAQ:AAPL`
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Ticker {
pub symbol: String,
@@ -77,6 +83,7 @@ impl Ticker {
tickers.iter().map(|t| Self::from(t, market)).collect()
}
+ /// 🤠 » returns the key of the ticker
pub fn key(&self) -> String {
format!("{}:{}", self.market, self.symbol)
}
@@ -123,17 +130,17 @@ pub trait RustlerAccessor {
#[async_trait]
pub trait Rustler: RustlerAccessor + Send + Sync {
// #region Unimplemented trait functions
- /// fn called after tickers are added to the rustler
+ /// 🤠 » fn called after tickers are added to the rustler
fn on_add(&mut self, tickers: &[Ticker]) -> Result<()>;
- /// fn called after tickers are deleted from the rustler
+ /// 🤠 » fn called after tickers are deleted from the rustler
fn on_delete(&mut self, tickers: &[Ticker]) -> Result<()>;
- /// connects the rustler to the data source
+ /// 🤠 » connects the rustler to the data source
async fn connect(&mut self) -> Result<()>;
- /// disconnects the rustler from the data source
+ /// 🤠 » disconnects the rustler from the data source
async fn disconnect(&mut self) -> Result<()>;
// #endregion
- /// should be called at construction time
+ /// 🤠 » starts the rustler
async fn start(&mut self) -> Result<()> {
let opts = self.opts();
if opts.connect_on_start {
@@ -142,7 +149,7 @@ pub trait Rustler: RustlerAccessor + Send + Sync {
Ok(())
}
- /// updates last stop and last run times and calls the appropriate callback
+ /// 🤠 » updates last stop and last run times and calls the appropriate callback
///
/// should be called after the status of the rustler changes
fn handle_status_change(&mut self) -> Result<()> {
@@ -333,7 +340,11 @@ macro_rules! rustler_accessors {
};
}
-// Define the macro
+/// **🤠 » rustler builder macro**
+///
+/// The `rustler!` macro is used to define a new `Rustler` struct, expanding the struct definition
+/// with the required fields and derives, and implementing the `RustlerAccessor` trait for the
+/// struct.
#[macro_export]
macro_rules! rustler {
// Entry point for the macro, takes the struct definition
diff --git a/rustlers/src/rustlerjar.rs b/lib/rustlers/rustlerjar.rs
similarity index 95%
rename from rustlers/src/rustlerjar.rs
rename to lib/rustlers/rustlerjar.rs
index 3283c52..cad59ba 100644
--- a/rustlers/src/rustlerjar.rs
+++ b/lib/rustlers/rustlerjar.rs
@@ -1,6 +1,6 @@
use {
- crate::rustlers::Rustler,
- entities::market,
+ super::rustler::Rustler,
+ crate::entities::market,
std::{collections::HashMap, sync::Arc},
tokio::sync::Mutex,
};
@@ -34,7 +34,7 @@ macro_rules! rustlerjar {
instances.push(instance);
)*
- $crate::rustlerjar::RustlerJar::new(instances, mappings)
+ $crate::rustlers::rustlerjar::RustlerJar::new(instances, mappings)
}};
}
diff --git a/rustlers/src/svc.rs b/lib/rustlers/svc.rs
similarity index 91%
rename from rustlers/src/svc.rs
rename to lib/rustlers/svc.rs
index 8b9b7cd..26e2cc8 100644
--- a/rustlers/src/svc.rs
+++ b/lib/rustlers/svc.rs
@@ -1,5 +1,9 @@
use {
- entities::{market, sea_orm::DatabaseConnection, ticker},
+ super::{
+ rustler::{Rustler, Ticker},
+ rustlerjar::RustlerJar,
+ },
+ crate::entities::{market, sea_orm::DatabaseConnection, ticker},
eyre::Result,
lool::{
logger::{info, warn},
@@ -12,11 +16,6 @@ use {
tokio::sync::Mutex,
};
-use crate::{
- rustlerjar::RustlerJar,
- rustlers::{Rustler, Ticker},
-};
-
// interface MarketExecData {
// entity: MarketModel;
// startJob?: Job;
@@ -34,6 +33,9 @@ use crate::{
// stopJob?: Job,
// }
+/// **🤠 » Rustlers Service**
+///
+/// The `RustlersSvc` is a service that manages the rustlers and orchestrates their executions.
pub struct RustlersSvc {
market_svc: market::Service,
sched: Scheduler,
@@ -41,7 +43,16 @@ pub struct RustlersSvc {
}
impl RustlersSvc {
+ /// **🤠 » create service**
+ ///
/// creates a new instance of the `RustlersSvc`
+ ///
+ /// **Arguments**
+ /// - `conn` - the database connection that will be used to get market and tickers data
+ /// - `rustlers` - the rustlers to be used by the service
+ ///
+ /// **Returns**
+ /// the created `RustlersSvc` instance
pub async fn new(conn: DatabaseConnection, rustlers: RustlerJar) -> Self {
let market_svc = market::Service::new(conn).await;
let sched = Scheduler::new();
@@ -53,6 +64,8 @@ impl RustlersSvc {
}
}
+ /// **🤠 » start rustlers**
+ ///
/// gets market data from the the database and starts
/// the corresponding rustler for each market
pub async fn start(&mut self) -> Result<()> {
@@ -66,6 +79,8 @@ impl RustlersSvc {
Ok(())
}
+ /// **🤠 » restart rustlers**
+ ///
/// stops all rustlers and then starts them again
pub async fn restart(&self) -> Result<()> {
// TODO: restart all rustlers; this method should clear everything we set up about the
@@ -222,6 +237,9 @@ impl RustlersSvc {
}
}
+/// creates a rule for the given time and offset
+///
+/// TODO: handle timezones
fn make_rule(
base: &RecurrenceRuleSet,
time: (u32, u32, u32),
@@ -239,6 +257,7 @@ fn make_rule(
rule
}
+/// the operation to be performed on the time (addition or subtraction)
enum Op {
Add,
Sub,
diff --git a/lib/socket/mod.rs b/lib/socket/mod.rs
new file mode 100644
index 0000000..78decec
--- /dev/null
+++ b/lib/socket/mod.rs
@@ -0,0 +1 @@
+// TODO: websocket gateway
diff --git a/migration/Cargo.toml b/migration/Cargo.toml
deleted file mode 100644
index e914b96..0000000
--- a/migration/Cargo.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[package]
-name = "migration"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "migration"
-path = "src/lib.rs"
-
-[dependencies]
-async-std = { version = "1", features = ["attributes", "tokio1"] }
-entities = { path = "../entities" }
-
-[dependencies.sea-orm-migration]
-version = "0.12.15"
-features = [
- "runtime-tokio-native-tls",
- "sqlx-sqlite",
-]
diff --git a/migration/README.md b/migration/README.md
deleted file mode 100644
index 3b438d8..0000000
--- a/migration/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# Running Migrator CLI
-
-- Generate a new migration file
- ```sh
- cargo run -- generate MIGRATION_NAME
- ```
-- Apply all pending migrations
- ```sh
- cargo run
- ```
- ```sh
- cargo run -- up
- ```
-- Apply first 10 pending migrations
- ```sh
- cargo run -- up -n 10
- ```
-- Rollback last applied migrations
- ```sh
- cargo run -- down
- ```
-- Rollback last 10 applied migrations
- ```sh
- cargo run -- down -n 10
- ```
-- Drop all tables from the database, then reapply all migrations
- ```sh
- cargo run -- fresh
- ```
-- Rollback all applied migrations, then reapply all migrations
- ```sh
- cargo run -- refresh
- ```
-- Rollback all applied migrations
- ```sh
- cargo run -- reset
- ```
-- Check the status of all migrations
- ```sh
- cargo run -- status
- ```
diff --git a/migration/src/lib.rs b/migration/src/lib.rs
deleted file mode 100644
index f5abbc6..0000000
--- a/migration/src/lib.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use sea_orm_migration::async_trait::async_trait;
-pub use sea_orm_migration::prelude::*;
-
-mod m20220101_000001_create_table_market;
-mod m20240325_200049_create_table_ticker;
-
-pub struct Migrator;
-
-#[async_trait]
-impl MigratorTrait for Migrator {
- fn migrations() -> Vec> {
- vec![
- Box::new(m20220101_000001_create_table_market::Migration),
- Box::new(m20240325_200049_create_table_ticker::Migration),
- ]
- }
-
- fn migration_table_name() -> sea_orm::DynIden {
- Alias::new("migrations").into_iden()
- }
-}
diff --git a/migration/src/main.rs b/migration/src/main.rs
deleted file mode 100644
index c6b6e48..0000000
--- a/migration/src/main.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-use sea_orm_migration::prelude::*;
-
-#[async_std::main]
-async fn main() {
- cli::run_cli(migration::Migrator).await;
-}
diff --git a/rustlers/Cargo.toml b/rustlers/Cargo.toml
deleted file mode 100644
index da76124..0000000
--- a/rustlers/Cargo.toml
+++ /dev/null
@@ -1,32 +0,0 @@
-[package]
-name = "rustlers"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "rustlers"
-path = "src/lib.rs"
-
-[dependencies]
-# workspace
-eyre = { workspace = true, default-features = false }
-tokio-tungstenite = { workspace = true }
-tokio = { workspace = true }
-lool = { workspace = true, features = [
- "logger",
- "sched.tokio",
- "sched.rule-recurrence",
- "macros",
-] }
-
-# internal
-entities = { path = "../entities" }
-
-# external
-chrono = "0.4.37"
-async-trait = "0.1.79"
-getset = "0.1.2"
-
-[build-dependencies]
-tonic-build = "0.11.0"
diff --git a/rustlers/src/lib.rs b/rustlers/src/lib.rs
deleted file mode 100644
index cc08e09..0000000
--- a/rustlers/src/lib.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-pub mod rustlerjar;
-pub mod rustlers;
-pub mod svc;
a [websocket gateway server](./lib/socket/) to stream stock pricing data to subscribed clients
+
+> [!NOTE]
+>
+> This library defines a _rustler_ as a service that scrapes stock pricing data for a
+> particular market.
+>
+> Although this library contains the core and abstract functionality for the rustlers, it doesn't include any concrete implementation for them.
+>
+> Actual concrete implementations for each market cannot be published for many reasons.
+
+## Example
+
+Check the [examples](./examples) directory for an example of how to use this library.
diff --git a/Taskfile.yaml b/Taskfile.yaml
index 26987e0..2a9109c 100644
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@@ -16,7 +16,7 @@ tasks:
run:watch:
desc: 🚀 watch rustler
cmds:
- - cargo watch -c -x "run"
+ - cargo watch -c -x "run --example=rustler"
build:watch:
desc: 🚀 watch rustler «build»
diff --git a/check_size.py b/check_size.py
deleted file mode 100644
index f88314a..0000000
--- a/check_size.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-small script to run after build, to check if there was a significant
-change on executable size, compared to the previous build.
-
-this aims to detect unwanted big differences before it's too late
-"""
-
-
-import os
-import pathlib
-
-curr_dir = pathlib.Path(os.getcwd())
-sizefile_path = pathlib.Path(curr_dir.joinpath('.task'))
-
-
-def bad(txt):
- return '\033[91m' + txt + '\033[0m'
-
-
-def good(txt):
- return '\033[92m' + txt + '\033[0m'
-
-
-def head(txt):
- return '\033[94m' + txt + '\033[0m'
-
-
-files = {
- 'release': curr_dir.joinpath('target/release/rustler.exe'),
- 'debug': curr_dir.joinpath('target/debug/rustler.exe'),
-}
-
-print("\n🧉 » exe file sizes change\n")
-
-for key, exe in files.items():
- if exe.is_file():
- sizefile = sizefile_path.joinpath(key)
- new_size: float = os.stat(exe).st_size / 1024
- old_size: float
-
- try:
- with open(sizefile, 'r') as f:
- old_size = float(f.read())
- except FileNotFoundError:
- old_size = 0
-
- # diff: str = f'{old_size:.0f}kb'
- diff = new_size - old_size
- diff_str = f"{'+' if diff > 0 else '=' if diff == 0 else ''}{diff:.0f}kb"
-
- fmt = bad if diff > 10 else good
-
- sizefile.parent.mkdir(parents=True, exist_ok=True)
-
- print(
- f'{head(key)}: {fmt(diff_str)} (prev: {old_size}kb, now: {fmt(f"{str(new_size)}kb")})'
- )
-
- with open(sizefile, 'w') as f:
- f.write(f'{new_size}')
diff --git a/entities/Cargo.toml b/entities/Cargo.toml
deleted file mode 100644
index a3ed977..0000000
--- a/entities/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "entities"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "entities"
-path = "src/lib.rs"
-
-[dependencies]
-# common
-eyre = { workspace = true, default-features = false }
-lool = { workspace = true, features = [ "cli", "cli.stylize", "macros", "logger" ] }
-
-# external
-sea-orm = { version = "0.12.15", features = [
- "runtime-tokio-native-tls",
- "sqlx-sqlite",
- "macros",
-] }
diff --git a/rustlers/src/rustlers/binance.rs b/examples/binance/mod.rs
similarity index 67%
rename from rustlers/src/rustlers/binance.rs
rename to examples/binance/mod.rs
index a34d249..ede5f57 100644
--- a/rustlers/src/rustlers/binance.rs
+++ b/examples/binance/mod.rs
@@ -1,33 +1,27 @@
use {
- crate::{
+ async_trait::async_trait,
+ eyre::Result,
+ lool::logger::info,
+ rustler_core::{
rustler,
rustlers::{Rustler, RustlerAccessor, RustlerStatus, Ticker},
},
- async_trait::async_trait,
- eyre::Result,
- lool::{cli::stylize::Stylize, logger::info},
std::collections::HashMap,
};
-const BINANCE_WSS_URL: &str = "wss://stream.binance.com:9443/stream";
-// const MANUAL_CLOSE_CODE: u16 = 4663;
-// const MANUAL_CLOSE_REASON: &str = "Manually Disconnected";
-
rustler!(
- /// 🤠 » **binance rustler**
- ///
- /// A rustler that steals quotes from Binance
- pub struct BinanceRustler {}
+ /// A fake rustler that does nothing but changing between different statuses.
+ pub struct FooRustler {}
);
-impl BinanceRustler {
+impl FooRustler {
pub fn create() -> impl Rustler {
Self::default()
}
}
#[async_trait]
-impl Rustler for BinanceRustler {
+impl Rustler for FooRustler {
async fn connect(&mut self) -> Result<()> {
if self.status == RustlerStatus::Connected || self.status == RustlerStatus::Connecting {
return Ok(());
@@ -35,10 +29,7 @@ impl Rustler for BinanceRustler {
self.set_status(RustlerStatus::Connecting)?;
- info!(
- "Connecting to Binance WSS: {}",
- BINANCE_WSS_URL.bright_green()
- );
+ info!("Connecting to data source");
self.set_status(RustlerStatus::Connected)?;
@@ -53,7 +44,7 @@ impl Rustler for BinanceRustler {
self.set_status(RustlerStatus::Disconnecting)?;
- info!("Disconnecting from Binance WSS");
+ info!("Disconnecting from data source");
self.set_status(RustlerStatus::Disconnected)?;
diff --git a/app/main.rs b/examples/rustler.rs
similarity index 58%
rename from app/main.rs
rename to examples/rustler.rs
index d852476..ea40fb0 100644
--- a/app/main.rs
+++ b/examples/rustler.rs
@@ -1,24 +1,25 @@
+mod binance;
+
use {
+ binance::FooRustler,
dotenvy::dotenv,
- eyre::Result,
+ eyre::{set_hook, DefaultHandler, Result},
lool::logger::{info, ConsoleLogger, Level},
- rustlers::{rustlerjar, rustlers::binance::BinanceRustler, svc::RustlersSvc},
+ rustler_core::{entities::db::get_connection, grpc, rustlerjar, rustlers::svc::RustlersSvc},
tokio::join,
};
-// TODO: here we will trigger the start of both the grpc server and the websocket gateway
-// look at: https://github.com/hyperium/tonic/discussions/740
-
#[tokio::main]
async fn main() -> Result<()> {
+ set_hook(Box::new(DefaultHandler::default_with))?;
ConsoleLogger::default_setup(Level::Trace, "rustler")?;
dotenv()?;
- let conn = entities::db::get_connection().await?;
+ let conn = get_connection().await?;
let mut rustler = RustlersSvc::new(
conn.clone(),
rustlerjar! {
- "BINANCE" => BinanceRustler
+ "BINANCE" => FooRustler
},
)
.await;
diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml
deleted file mode 100644
index 04e8d20..0000000
--- a/gateway/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "gateway"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "gateway"
-path = "src/lib.rs"
-
-[dependencies]
-entities = { path = "../entities" }
-eyre = { workspace = true, default-features = false }
-
diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs
deleted file mode 100644
index e69de29..0000000
diff --git a/grpc/Cargo.toml b/grpc/Cargo.toml
deleted file mode 100644
index acd802c..0000000
--- a/grpc/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "grpc"
-version = "0.1.0"
-edition = "2021"
-publish = false
-
-[lib]
-name = "grpc"
-path = "src/lib.rs"
-
-[dependencies]
-# common
-lool = { workspace = true, features = ["logger"]}
-eyre = { workspace = true, default-features = false }
-
-# internal
-entities = { path = "../entities" }
-
-# external
-prost = "0.12.3"
-tonic = "0.11.0"
-
-[build-dependencies]
-tonic-build = "0.11.0"
-
diff --git a/grpc/build.rs b/lib/build.rs
similarity index 71%
rename from grpc/build.rs
rename to lib/build.rs
index 2d85665..1ac75f0 100644
--- a/grpc/build.rs
+++ b/lib/build.rs
@@ -1,5 +1,9 @@
fn main() {
- let proto_files = vec!["./proto/rustler.proto", "./proto/market.proto", "./proto/ticker.proto"];
+ let proto_files = vec![
+ "./lib/grpc/proto/rustler.proto",
+ "./lib/grpc/proto/market.proto",
+ "./lib/grpc/proto/ticker.proto",
+ ];
for proto_file in proto_files {
compile_proto(proto_file);
diff --git a/migration/src/m20220101_000001_create_table_market.rs b/lib/entities/migration/m20220101_000001_create_table_market.rs
similarity index 98%
rename from migration/src/m20220101_000001_create_table_market.rs
rename to lib/entities/migration/m20220101_000001_create_table_market.rs
index 45c2c2f..3af09fb 100644
--- a/migration/src/m20220101_000001_create_table_market.rs
+++ b/lib/entities/migration/m20220101_000001_create_table_market.rs
@@ -1,6 +1,7 @@
use sea_orm_migration::{async_trait::async_trait, prelude::*};
#[derive(DeriveMigrationName)]
+/// 🤠 » create table `market`
pub struct Migration;
#[async_trait]
diff --git a/migration/src/m20240325_200049_create_table_ticker.rs b/lib/entities/migration/m20240325_200049_create_table_ticker.rs
similarity index 97%
rename from migration/src/m20240325_200049_create_table_ticker.rs
rename to lib/entities/migration/m20240325_200049_create_table_ticker.rs
index d006c58..6fea9fa 100644
--- a/migration/src/m20240325_200049_create_table_ticker.rs
+++ b/lib/entities/migration/m20240325_200049_create_table_ticker.rs
@@ -1,6 +1,7 @@
use sea_orm_migration::{async_trait::async_trait, prelude::*};
#[derive(DeriveMigrationName)]
+/// 🤠 » create table `ticker`
pub struct Migration;
#[async_trait]
diff --git a/lib/entities/migration/mod.rs b/lib/entities/migration/mod.rs
new file mode 100644
index 0000000..9b9514c
--- /dev/null
+++ b/lib/entities/migration/mod.rs
@@ -0,0 +1,2 @@
+mod m20220101_000001_create_table_market;
+mod m20240325_200049_create_table_ticker;
diff --git a/entities/src/lib.rs b/lib/entities/mod.rs
similarity index 98%
rename from entities/src/lib.rs
rename to lib/entities/mod.rs
index d5ea9ea..7eaf2c5 100644
--- a/entities/src/lib.rs
+++ b/lib/entities/mod.rs
@@ -1,3 +1,4 @@
+pub mod migration;
pub use sea_orm;
mod orm {
diff --git a/entities/src/orm/market.rs b/lib/entities/orm/market.rs
similarity index 96%
rename from entities/src/orm/market.rs
rename to lib/entities/orm/market.rs
index 402f366..a04589d 100644
--- a/entities/src/orm/market.rs
+++ b/lib/entities/orm/market.rs
@@ -4,6 +4,7 @@ use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "market")]
+/// 🤠 » market entity model
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
diff --git a/entities/src/orm/ticker.rs b/lib/entities/orm/ticker.rs
similarity index 96%
rename from entities/src/orm/ticker.rs
rename to lib/entities/orm/ticker.rs
index 2e3e94c..92fda31 100644
--- a/entities/src/orm/ticker.rs
+++ b/lib/entities/orm/ticker.rs
@@ -4,6 +4,7 @@ use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "ticker")]
+/// 🤠 » ticker entity model
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
diff --git a/entities/src/services/market.rs b/lib/entities/services/market.rs
similarity index 83%
rename from entities/src/services/market.rs
rename to lib/entities/services/market.rs
index 72177b5..00acc94 100644
--- a/entities/src/services/market.rs
+++ b/lib/entities/services/market.rs
@@ -1,5 +1,5 @@
use {
- crate::{
+ crate::entities::{
market::{Entity as Market, Model as MarketModel},
ticker::{Entity as Ticker, Model as TickerModel},
},
@@ -7,6 +7,7 @@ use {
sea_orm::{DatabaseConnection, DbErr, EntityTrait, IntoActiveModel},
};
+/// 🤠 » service for the `Market` entity
pub struct Service {
conn: DatabaseConnection,
}
@@ -16,16 +17,19 @@ impl Service {
Self { conn }
}
+ /// 🤠 » gets all markets from the database
pub async fn get_all(&self) -> Result