Data in Motion - Drought Map

Read Time: 5 minutes

Visualizations like charts and graphs can be powerful tools, but they are often static. An even more powerful story can be told over time with animations and videos. Using F#, along with a couple tools, I’ll do just that. Today’s focus is on the Palmer Drought Severity data for the U.S. over that last one-hundred years. This is a lighter post, so hopefully the video is mesmerizing enough to compensate for any lack of depth.

How is this accomplished? I reach into F#’s bag of tricks to leverage Deedle, Plotly.NET, and ffmpeg in order to transform a series of data files into a singular video showing county-level drought data from 1900-2016. Together these bring static data into a dynamic representation. For reference, the Palmer Drought Severity Index (PDSI) typically ranges from -10 (dry) to 10 (wet). Putting this all together is pretty straight-forward, but I wanted to call out a couple specific parts. For this particular example Deedle is overkill, but pairing it with Plotly.NET can often be useful in more complex situations. Plotly offers some nice customization options, which I take advantage of below. Once all the images are generated with Plotly, F# can shell out to ffmpeg to perform the video assembly. I do this in two parts, creating both an mp4 and webm file.

Source Data: Centers for Disease Control and Prevention. National Environmental Public Health Tracking Network. https://data.cdc.gov/Environmental-Health-Toxicology/Palmer-Drought-Severity-Index-1895-2016/en5r-5ds4

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
open System
open System.Diagnostics
open System.IO
open Deedle
open Newtonsoft.Json
open Plotly.NET
open Plotly.NET.ImageExport

/// Convert a datafile into an imagefile name (with no extension)
let buildImageNameNoExtension i _dataFileName =
  Path.Combine("images", sprintf "image_%04d" i)

/// Convert a datafile name into a year-month chart title
let fileNameToTitle dataFileName =
  let regex = Text.RegularExpressions.Regex("drought_(\d+)_(\d+).csv")

  let matches = regex.Match(dataFileName)
  let year = matches.Groups.[1].Captures.[0].ToString() |> int
  let month = matches.Groups.[2].Captures.[0].ToString() |> int

  sprintf "%4d-%02d" year month

/// Json object of county code to map coordinates polygon
/// Source: https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json
let geoJson =
  IO.File.ReadAllText("data/geojson-counties-fips.json")
  |> JsonConvert.DeserializeObject

/// Build map of drought data
let buildMap index dataFile =
  let title = fileNameToTitle dataFile

  let data = Frame.ReadCsv(dataFile, false, separators = ",")

  let fips =
    data
    |> Frame.getCol "Column4" // "fips"
    |> Series.values
    |> Array.ofSeq

  let pdsi =
    data
    |> Frame.getCol "Column5" // "pdsi"
    |> Series.values
    |> Array.ofSeq

  let chart =
    Chart.ChoroplethMap (
      locations = fips,
      z = pdsi,
      Locationmode = StyleParam.LocationFormat.GeoJson_Id,
      GeoJson = geoJson,
      FeatureIdKey = "id",
      Colorscale = 
        StyleParam.Colorscale.Custom([
          (0.0 , "#5d0c06")
          (0.25, "#8d0c06")
          (0.5 , "#dedede")
          (0.75, "#060c8d")
          (1.0 , "#060c5d") ]),
      Zmin = -10.0,
      Zmax = 10.0)
    |> Chart.withMap (Geo.init (Scope = StyleParam.GeoScope.Usa))
    |> Chart.withColorBarStyle ("PDSI", Length = 0.75)
    |> Chart.withTitle (title=title, Titlefont=Font.init(Family=StyleParam.FontFamily.Courier_New, Size=32.))
    |> Chart.withSize (800., 500.)
    |> Chart.savePNG (path = buildImageNameNoExtension index dataFile, Width = 800, Height = 500)

/// Execute command
let exec command args =
  let startInfo = ProcessStartInfo(FileName = command, Arguments = args)
  let p = new Process(StartInfo = startInfo)

  let success = p.Start()
  if not success then
    printfn "Process Failed"
  else 
    p.WaitForExit()

/// Build a video (mp4) using all pngs in the sourceDir
let buildVideo sourceDir dstFile =
  exec "ffmpeg" $"-y -i {sourceDir}/image_%%04d.png -c:v libx264 -vf fps=120 -pix_fmt yuv420p {dstFile}"

/// Convert an mp4 to a different file format (i.e. webm or .gif)
let convertVideo (inputFile: string) (outputFile: string) =
  exec "ffmpeg" $"-i {inputFile} {outputFile}"

[<EntryPoint>]
let main argv =
  // Create map images for each month of the data series
  // Name the images numerically, for consumption by ffmpeg  
  IO.Directory.GetFiles("./data", "drought*.csv")
  |> Array.sort
  |> Array.mapi (fun i x -> (i, x))
  |> Array.iter (fun (i, x) -> buildMap i x)

  // Combine images into a video
  buildVideo "images" "drought.mp4" |> ignore
  convertVideo "drought.mp4" "drought.webm" |> ignore

  0
Related Posts: