It was great to join @christianminich.com in this webinar. He shared a lot of great insights in how he's using AI for data engineering.
colton
@colton.boo
2.4K followers 1.9K following 95 posts
Data Engineering / Music / Photography / Tinkering cmpadden.github.io
![# make sure these packages are installed
library(reticulate)
library(readr)
library(glue)
library(magrittr)
# Ensure that RETICULATE_PYTHON is set to the poetry path
reticulate::py_config()
stopifnot(reticulate::py_module_available("dagster_pipes"))
# Function to convert R types to Python types
convert_r_to_python_types <- function(df) {
# Get R types
r_types <- sapply(df, class)
# Define type mapping
type_mapping <- list(
"numeric" = "float",
"integer" = "int",
"character" = "str",
"factor" = "str",
"logical" = "bool",
"Date" = "datetime.date",
"POSIXct" = "datetime.datetime",
"POSIXlt" = "datetime.datetime"
)
# Convert types
python_types <- sapply(r_types, function(x) {
if (x %in% names(type_mapping)) {
type_mapping[[x]]
} else {
"object" # default type
}
})
return(reticulate::r_to_py(as.list(python_types)))
}
# Import Python modules
# R doesn't support selective imports like Python, so you have to do this
# to avoid typing the full namespace path repeatedly...
os <- reticulate::import("os")
dagster_pipes <- reticulate::import("dagster_pipes")
open_dagster_pipes <- dagster_pipes$open_dagster_pipes
PipesContext <- dagster_pipes$PipesContext
# https://rstudio.github.io/reticulate/articles/calling_python.html#with-contexts
with(open_dagster_pipes() %as% pipes, {
context <- PipesContext$get()
context$log$info("This is a log from R!")
data(iris)
context$log$info(head(iris))
context$log$info(os$environ["MY_ENV_VAR_IN_SUBPROCESS"])
output_dir <- Sys.getenv("OUTPUT_DIR")
iris_head <- head(iris)
context$log$info(glue::glue("output_dir: {output_dir}"))
#python function to report back the materialization and metadata
context$report_custom_message(
payload = reticulate::r_to_py(list(
"dagster/row_count" = nrow(iris),
# if using report_asset_mater…](https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:lc5jzrr425fyah724df3z5ik/bafkreie4jgubv762s7epk3rdepyc7iww276iq7f75metklqwsqwt3yt23e)
