10

Data in Motion - Drought Map

 3 years ago
source link: https://www.codesuji.com/2021/07/28/Data-in-Motion-Drought/
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.
neoserver,ios ssh client

Data in Motion - Drought Map

Read Time: 5 minutes

Visualizations like charts and graphs can be powerful tools, but they are often static. An even more powerful story can be told over time with animations and videos. Using F#, along with a couple tools, I’ll do just that. Today’s focus is on the Palmer Drought Severity data for the U.S. over that last one-hundred years. This is a lighter post, so hopefully the video is mesmerizing enough to compensate for any lack of depth.



How is this accomplished? I reach into F#’s bag of tricks to leverage Deedle, Plotly.NET, and ffmpeg in order to transform a series of data files into a singular video showing county-level drought data from 1900-2016. Together these bring static data into a dynamic representation. For reference, the Palmer Drought Severity Index (PDSI) typically ranges from -10 (dry) to 10 (wet). Putting this all together is pretty straight-forward, but I wanted to call out a couple specific parts. For this particular example Deedle is overkill, but pairing it with Plotly.NET can often be useful in more complex situations. Plotly offers some nice customization options, which I take advantage of below. Once all the images are generated with Plotly, F# can shell out to ffmpeg to perform the video assembly. I do this in two parts, creating both an mp4 and webm file.

open System
open System.Diagnostics
open System.IO
open Deedle
open Newtonsoft.Json
open Plotly.NET
open Plotly.NET.ImageExport

/// Convert a datafile into an imagefile name (with no extension)
let buildImageNameNoExtension i _dataFileName =
  Path.Combine("images", sprintf "image_%04d" i)

/// Convert a datafile name into a year-month chart title
let fileNameToTitle dataFileName =
  let regex = Text.RegularExpressions.Regex("drought_(\d+)_(\d+).csv")

let matches = regex.Match(dataFileName)
  let year = matches.Groups.[1].Captures.[0].ToString() |> int
  let month = matches.Groups.[2].Captures.[0].ToString() |> int

sprintf "%4d-%02d" year month

/// Json object of county code to map coordinates polygon
/// Source: https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json
let geoJson =
  IO.File.ReadAllText("data/geojson-counties-fips.json")
  |> JsonConvert.DeserializeObject

/// Build map of drought data
let buildMap index dataFile =
  let title = fileNameToTitle dataFile

let data = Frame.ReadCsv(dataFile, false, separators = ",")

let fips =
    data
    |> Frame.getCol "Column4" // "fips"
    |> Series.values
    |> Array.ofSeq

let pdsi =
    data
    |> Frame.getCol "Column5" // "pdsi"
    |> Series.values
    |> Array.ofSeq

let chart =
    Chart.ChoroplethMap (
      locations = fips,
      z = pdsi,
      Locationmode = StyleParam.LocationFormat.GeoJson_Id,
      GeoJson = geoJson,
      FeatureIdKey = "id",
      Colorscale = 
        StyleParam.Colorscale.Custom([
          (0.0 , "#5d0c06")
          (0.25, "#8d0c06")
          (0.5 , "#dedede")
          (0.75, "#060c8d")
          (1.0 , "#060c5d") ]),
      Zmin = -10.0,
      Zmax = 10.0)
    |> Chart.withMap (Geo.init (Scope = StyleParam.GeoScope.Usa))
    |> Chart.withColorBarStyle ("PDSI", Length = 0.75)
    |> Chart.withTitle (title=title, Titlefont=Font.init(Family=StyleParam.FontFamily.Courier_New, Size=32.))
    |> Chart.withSize (800., 500.)
    |> Chart.savePNG (path = buildImageNameNoExtension index dataFile, Width = 800, Height = 500)

/// Execute command
let exec command args =
  let startInfo = ProcessStartInfo(FileName = command, Arguments = args)
  let p = new Process(StartInfo = startInfo)

let success = p.Start()
  if not success then
    printfn "Process Failed"
  else 
    p.WaitForExit()

/// Build a video (mp4) using all pngs in the sourceDir
let buildVideo sourceDir dstFile =
  exec "ffmpeg" $"-y -i {sourceDir}/image_%%04d.png -c:v libx264 -vf fps=120 -pix_fmt yuv420p {dstFile}"

/// Convert an mp4 to a different file format (i.e. webm or .gif)
let convertVideo (inputFile: string) (outputFile: string) =
  exec "ffmpeg" $"-i {inputFile} {outputFile}"

[<EntryPoint>]
let main argv =
  // Create map images for each month of the data series
  // Name the images numerically, for consumption by ffmpeg  
  IO.Directory.GetFiles("./data", "drought*.csv")
  |> Array.sort
  |> Array.mapi (fun i x -> (i, x))
  |> Array.iter (fun (i, x) -> buildMap i x)

// Combine images into a video
  buildVideo "images" "drought.mp4" |> ignore
  convertVideo "drought.mp4" "drought.webm" |> ignore

0

Share


report

Recommend

About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK