Data in Motion - Population Map

Read Time: 4 minutes

Today’s “data in motion” post is a quick population over time visualization. I’ll use U.S. census data over the last one-hundred-ish years. As is the theme in this series, I’ll convert the raw data into a video of the data over time using primarily F#.

The above video represents state-level population data from 1910 to 2020, in decade increments. The static data is brought together using a combination of tools. The primary one being F#, along with the libraries Deedle and Plotly.NET for data manipulation and chart creation (respectively). It is pulled together using ffmpeg in order to transform a series of images into a final video.

Source Data: U.S. Census Bureau. https://www.census.gov/data/tables/time-series/dec/popchange-data-text.html

For posterity sake, here are the package versions. Plotly.NET has made a lot of great progress lately.

1
2
3
dotnet add package Deedle --version 2.5.0
dotnet add package Plotly.NET --version 2.0.0
dotnet add package Plotly.NET.ImageExport --version 2.0.0

And here is the code. As you can imagine, it is similar to the other posts in this series. The changes are mostly a result of different data formats and newer Plotly.NET versions.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
open System
open System.Diagnostics
open Deedle
open Plotly.NET
open Plotly.NET.ImageExport

/// Execute command
let exec command args =
  let startInfo = ProcessStartInfo(FileName = command, Arguments = args)
  let p = new Process(StartInfo = startInfo)

  let success = p.Start()
  if not success then
    printfn "Process Failed"
  else 
    p.WaitForExit()

/// Build a video (mp4) using all pngs in the sourceDir
let buildVideo sourceDir dstFile =
  exec "ffmpeg" $"-y -framerate 1 -i {sourceDir}/image_%%04d.png -c:v libx264 -r 1 -pix_fmt yuv420p {dstFile}"

/// Convert an mp4 to a different file format (i.e. webm or .gif)
let convertVideo (inputFile: string) (outputFile: string) =
  exec "ffmpeg" $"-i {inputFile} {outputFile}"

[<EntryPoint>]
let main argv =
  let imageDir = "../images/"

  // Used to convert state names to abbreviations
  let stateLookup =
    Frame.ReadCsv("../data/states.csv", true, separators = ",")
    |> Frame.indexRows "Name"
    |> Frame.getCol "Code"
    |> Series.observations
    |> Map.ofSeq

  // Population data
  let data = Frame.ReadCsv("../data/population.csv", true, separators = ",")

  // Get all years in the file
  let years =
    data
    |> Frame.getCol "Year"
    |> Series.values
    |> Seq.distinct
    |> Seq.mapi (fun i x -> (i, x))

  for (index, year) in years do
    // Build an image for each year in file

    // State abbreviations
    let statesForYear =
      data
      |> Frame.filterRowValues (fun row -> row.GetAs<string>("Year") = year)
      |> Frame.getCol "Name"
      |> Series.values
      |> Seq.map (fun state -> stateLookup.Item state)

    // State populations
    let populationsForYear =
      data
      |> Frame.filterRowValues (fun row -> row.GetAs<string>("Year") = year)
      |> Frame.getCol "Resident Population"
      |> Series.values

    // Build chart
    Chart.ChoroplethMap (
      locations = statesForYear,
      z = populationsForYear,
      LocationMode = StyleParam.LocationFormat.USA_states,
      FeatureIdKey = "id",
      ColorScale = 
        StyleParam.Colorscale.Custom([
          (0., Color.fromHex("#00ca00"))
          (1., Color.fromHex("#002400"))
        ])
    |> Chart.withGeoStyle(Scope = StyleParam.GeoScope.Usa)
    |> Chart.withColorBarStyle ("Population", Len = 0.75)
    |> Chart.withTitle (title=$"{year} population",
                        TitleFont=Font.init(Family=StyleParam.FontFamily.Courier_New, Size=32.))
    |> Chart.withSize (800., 500.)
    |> Chart.savePNG (path = (sprintf "%s/image_%04d" imageDir index),
                      Width = 800,
                      Height = 500)

  // Convert images to a video
  buildVideo "../images" "population.mp4" |> ignore
  convertVideo "population.mp4" "population.webm" |> ignore

  0
Related Posts: