Data in Motion - Precipitation Map

Read Time: 4 minutes

Today is again a lighter post playing with visualizations. The data focus is on the Standardized Precipitation Index data for the U.S. over that last one-hundred years. Static images and data can be useful, but visualizing data over time can be a welcome addition for analysis. So I’ll be converting data ultimately into a video of the data over time using primarily F#.

The above video starts with the data. In its raw form it is county-level precipitation data from 1895-2016. More specifically, the Standardized Precipitation Index (SPI) is a standard deviation metric, and for this dataset values range from -3 to 3. So this shows over time is how much areas deviate from the relative mean. The static data is brought together using a combination of tools. The primary one being F#, along with the libraries Deedle and Plotly.NET for data manipulation and chart creation (respectively). It is pulled together using ffmpeg in order to transform a series of images into a final video.

Source Data: Centers for Disease Control and Prevention. National Environmental Public Health Tracking Network. https://data.cdc.gov/Environmental-Health-Toxicology/Standardized-Precipitation-Index-1895-2016/xbk2-5i4e

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
open System
open System.Diagnostics
open System.IO
open Deedle
open Newtonsoft.Json
open Plotly.NET
open Plotly.NET.ImageExport

/// Convert a datafile into an imagefile name (with no extension)
let buildImageNameNoExtension i _dataFileName =
Path.Combine("images", sprintf "image_%04d" i)

/// Convert a datafile name into a year-month chart title
let fileNameToTitle dataFileName =
let regex = Text.RegularExpressions.Regex("precipitation_(\d+)_(\d+).csv")

let matches = regex.Match(dataFileName)
let year = matches.Groups.[1].Captures.[0].ToString() |> int
let month = matches.Groups.[2].Captures.[0].ToString() |> int

sprintf "%4d-%02d" year month

/// Json object of county code to map coordinates polygon
/// Source: https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json
let geoJson =
IO.File.ReadAllText("data/geojson-counties-fips.json")
|> JsonConvert.DeserializeObject

/// Build map of precipitation data
let buildMap index dataFile =
let title = fileNameToTitle dataFile

let data = Frame.ReadCsv(dataFile, true, separators = ",")

let fips =
data
|> Frame.getCol "countyfips"
|> Series.values
|> Array.ofSeq

let spi =
data
|> Frame.getCol "spi"
|> Series.values
|> Array.ofSeq

let chart =
Chart.ChoroplethMap (
locations = fips,
z = spi,
Locationmode = StyleParam.LocationFormat.GeoJson_Id,
GeoJson = geoJson,
FeatureIdKey = "id",
Colorscale =
StyleParam.Colorscale.Custom([
(0.0, "#3d0c06")
(0.165, "#6d0c06")
(0.25, "#9d0c06")
(0.5 , "#9a9a9a")
(0.665, "#060c9d")
(0.75, "#060c6d")
(1.0, "#060c3d")
]),
Zmin = -3,
Zmax = 3)
|> Chart.withMap (Geo.init (Scope = StyleParam.GeoScope.Usa))
|> Chart.withColorBarStyle ("SPI", Length = 0.75)
|> Chart.withTitle (title=title, Titlefont=Font.init(Family=StyleParam.FontFamily.Courier_New, Size=32.))
|> Chart.withSize (800., 500.)
|> Chart.savePNG (path = buildImageNameNoExtension index dataFile, Width = 800, Height = 500)

/// Execute command
let exec command args =
let startInfo = ProcessStartInfo(FileName = command, Arguments = args)
let p = new Process(StartInfo = startInfo)

let success = p.Start()
if not success then
printfn "Process Failed"
else
p.WaitForExit()

/// Build a video (mp4) using all pngs in the sourceDir
let buildVideo sourceDir dstFile =
exec "ffmpeg" $"-y -i {sourceDir}/image_%%04d.png -c:v libx264 -vf fps=5 -pix_fmt yuv420p {dstFile}"

/// Convert an mp4 to a different file format (i.e. webm or .gif)
let convertVideo (inputFile: string) (outputFile: string) =
exec "ffmpeg" $"-i {inputFile} {outputFile}"

[<EntryPoint>]
let main argv =
// Create map images for each month of the data series
// Name the images numerically, for consumption by ffmpeg
IO.Directory.GetFiles("./data", "precipitation*.csv")
|> Array.sort
|> Array.mapi (fun i x -> (i, x))
|> Array.iter (fun (i, x) -> buildMap i x)

// Combine images into a video
buildVideo "images" "precipitation.mp4" |> ignore
convertVideo "precipitation.mp4" "precipitation.webm" |> ignore

0