Microservice Usage Logging with Openresty and Google BigQuery

log_by_lua_block {
local payload = {
bytes = tonumber(ngx.var.bytes_sent),
status = tonumber(ngx.var.status),
timestamp = ngx.now()
... -- more values
}
-- post payload to favorite backend for timeseries analysis
}
lua_shared_dict usage_logging 10m;init_worker_by_lua_block {
local Logging = require "descarteslabs.logging"
l = Logging.new()
l:watch(ngx.shared.usage_logging)
}
location = /test {
proxy_pass service;
log_by_lua_block {
local payload = {
bytes = tonumber(ngx.var.bytes_sent),
status = tonumber(ngx.var.status),
timestamp = ngx.now()
... -- more values
}
local Logging = require "descarteslabs.logging"
l = Logging.save(ngx.shared.usage_logging, payload)
}
}
local check_function
check_function = function(premature)
if not premature then
while
true do
local
requests = self:get(size)
pcall(_M.save, self, rows)

if #rows < size then
break
end
end

local
ok, err = ngx.timer.at(delay(), check_function)
if not ok then
log(ERR, "failed to create timer: ", err)
return
end
end
end
SELECT 
COUNT(*) as calls,
SUM(bytes) as bytes,
DATETIME_TRUNC(DATETIME(timestamp), `second` ) as w
FROM `project.dataset.table`
WHERE
status=200
GROUP BY
w
ORDER BY
w desc
LIMIT 100
Google BigQuery Results Using Vegeta for Load Testing

--

--

--

Now writing at: https://justin.poehnelt.com — Senior Developer Relations Engineer @google

Love podcasts or audiobooks? Learn on the go with our new app.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Justin Poehnelt

Justin Poehnelt

Now writing at: https://justin.poehnelt.com — Senior Developer Relations Engineer @google

More from Medium

Can we ETL with Kafka?

Using the Python Google Drive API to Upload Files from Github Actions

Write error logs from Composer and create an alert policy on Stackdriver

How to Automate Dataset Comparison Using Terraform And BigQuery