Leon4gr45 commited on
Commit
5180a84
·
verified ·
1 Parent(s): ab4d818

Delete test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs with huggingface_hub

Browse files
test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs DELETED
@@ -1,1134 +0,0 @@
1
- defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
2
- use PlausibleWeb.ConnCase, async: false
3
- use Plausible.Repo
4
-
5
- @default_reports %{
6
- "visitors.csv" => %{
7
- dimensions: ["time:day"],
8
- metrics: [
9
- "visitors",
10
- "pageviews",
11
- "visits",
12
- "views_per_visit",
13
- "bounce_rate",
14
- "visit_duration"
15
- ]
16
- },
17
- "sources.csv" => %{
18
- dimensions: ["visit:source"],
19
- metrics: ["visitors", "bounce_rate", "visit_duration"]
20
- },
21
- "channels.csv" => %{
22
- dimensions: ["visit:channel"],
23
- metrics: ["visitors", "bounce_rate", "visit_duration"]
24
- },
25
- "referrers.csv" => %{
26
- dimensions: ["visit:referrer"],
27
- metrics: ["visitors", "bounce_rate", "visit_duration"]
28
- },
29
- "utm_mediums.csv" => %{
30
- dimensions: ["visit:utm_medium"],
31
- always_on_filters: [["is_not", "visit:utm_medium", [""]]],
32
- metrics: ["visitors", "bounce_rate", "visit_duration"]
33
- },
34
- "utm_sources.csv" => %{
35
- dimensions: ["visit:utm_source"],
36
- always_on_filters: [["is_not", "visit:utm_source", [""]]],
37
- metrics: ["visitors", "bounce_rate", "visit_duration"]
38
- },
39
- "utm_campaigns.csv" => %{
40
- dimensions: ["visit:utm_campaign"],
41
- always_on_filters: [["is_not", "visit:utm_campaign", [""]]],
42
- metrics: ["visitors", "bounce_rate", "visit_duration"]
43
- },
44
- "utm_contents.csv" => %{
45
- dimensions: ["visit:utm_content"],
46
- always_on_filters: [["is_not", "visit:utm_content", [""]]],
47
- metrics: ["visitors", "bounce_rate", "visit_duration"]
48
- },
49
- "utm_terms.csv" => %{
50
- dimensions: ["visit:utm_term"],
51
- always_on_filters: [["is_not", "visit:utm_term", [""]]],
52
- metrics: ["visitors", "bounce_rate", "visit_duration"]
53
- },
54
- "pages.csv" => %{
55
- dimensions: ["event:page"],
56
- metrics: ["visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"]
57
- },
58
- "entry_pages.csv" => %{
59
- dimensions: ["visit:entry_page"],
60
- always_on_filters: [["is_not", "visit:entry_page", [""]]],
61
- metrics: ["visitors", "visits", "bounce_rate", "visit_duration"]
62
- },
63
- "exit_pages.csv" => %{
64
- dimensions: ["visit:exit_page"],
65
- always_on_filters: [["is_not", "visit:exit_page", [""]]],
66
- metrics: ["visitors", "visits", "exit_rate"]
67
- },
68
- "browsers.csv" => %{
69
- dimensions: ["visit:browser"],
70
- metrics: ["visitors"]
71
- },
72
- "browser_versions.csv" => %{
73
- dimensions: ["visit:browser_version", "visit:browser"],
74
- metrics: ["visitors"]
75
- },
76
- "operating_systems.csv" => %{
77
- dimensions: ["visit:os"],
78
- metrics: ["visitors"]
79
- },
80
- "operating_system_versions.csv" => %{
81
- dimensions: ["visit:os_version", "visit:os"],
82
- metrics: ["visitors"]
83
- },
84
- "devices.csv" => %{
85
- dimensions: ["visit:device"],
86
- metrics: ["visitors"]
87
- },
88
- "countries.csv" => %{
89
- dimensions: ["visit:country_name"],
90
- always_on_filters: [["is_not", "visit:country", ["\0\0", "ZZ"]]],
91
- metrics: ["visitors"]
92
- },
93
- "regions.csv" => %{
94
- dimensions: ["visit:region_name"],
95
- always_on_filters: [["is_not", "visit:region", [""]]],
96
- metrics: ["visitors"]
97
- },
98
- "cities.csv" => %{
99
- dimensions: ["visit:city_name"],
100
- always_on_filters: [["is_not", "visit:city", [0]]],
101
- metrics: ["visitors"]
102
- },
103
- "custom_props.csv" => %{
104
- dimensions: ["event:props:*"],
105
- metrics: ["visitors", "events", "percentage"]
106
- },
107
- "conversions.csv" => %{
108
- dimensions: ["event:goal"],
109
- metrics: ["visitors", "events"]
110
- }
111
- }
112
-
113
- @page_filtered_reports put_in(@default_reports, ["visitors.csv", :metrics], [
114
- "visitors",
115
- "pageviews",
116
- "visits",
117
- "bounce_rate",
118
- "time_on_page",
119
- "scroll_depth"
120
- ])
121
- |> put_in(
122
- ["exit_pages.csv", :metrics],
123
- get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]
124
- )
125
- @prop_filtered_reports put_in(
126
- @default_reports,
127
- ["exit_pages.csv", :metrics],
128
- get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]
129
- )
130
-
131
- @goal_filtered_reports Map.new(@default_reports, fn
132
- {"visitors.csv" = filename, params} ->
133
- {filename,
134
- %{params | metrics: ["visitors", "events", "group_conversion_rate"]}}
135
-
136
- {"custom_props.csv" = filename, params} ->
137
- {filename,
138
- %{params | metrics: ["visitors", "events", "conversion_rate"]}}
139
-
140
- {"conversions.csv" = filename, params} ->
141
- {filename, params}
142
-
143
- {filename, params} ->
144
- {filename,
145
- %{params | metrics: ["visitors", "group_conversion_rate"]}}
146
- end)
147
-
148
- @base_params %{
149
- date_range: "30d",
150
- relative_date: nil,
151
- filters: [],
152
- include: %{imports: false},
153
- reports: @default_reports
154
- }
155
-
156
- describe "POST /api/stats/:domain/export" do
157
- setup [:create_user, :create_site, :log_in]
158
-
159
- test "exports all the necessary CSV files", %{conn: conn, site: site} do
160
- conn = do_export(conn, site, @base_params)
161
-
162
- assert {"content-type", "application/zip; charset=utf-8"} =
163
- List.keyfind(conn.resp_headers, "content-type", 0)
164
-
165
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
166
-
167
- zip = Enum.map(zip, fn {filename, _} -> filename end)
168
-
169
- assert ~c"visitors.csv" in zip
170
- assert ~c"browsers.csv" in zip
171
- assert ~c"browser_versions.csv" in zip
172
- assert ~c"cities.csv" in zip
173
- assert ~c"conversions.csv" in zip
174
- assert ~c"countries.csv" in zip
175
- assert ~c"devices.csv" in zip
176
- assert ~c"entry_pages.csv" in zip
177
- assert ~c"exit_pages.csv" in zip
178
- assert ~c"operating_systems.csv" in zip
179
- assert ~c"operating_system_versions.csv" in zip
180
- assert ~c"pages.csv" in zip
181
- assert ~c"regions.csv" in zip
182
- assert ~c"sources.csv" in zip
183
- assert ~c"channels.csv" in zip
184
- assert ~c"utm_campaigns.csv" in zip
185
- assert ~c"utm_contents.csv" in zip
186
- assert ~c"utm_mediums.csv" in zip
187
- assert ~c"utm_sources.csv" in zip
188
- assert ~c"utm_terms.csv" in zip
189
- end
190
-
191
- test "limits pages.csv and exit_pages.csv to 100 rows", %{conn: conn, site: site} do
192
- events = for i <- 1..101, do: build(:pageview, pathname: "/page-#{i}")
193
- populate_stats(site, events)
194
-
195
- zip_content =
196
- conn
197
- |> do_export(site, %{@base_params | date_range: "day"})
198
- |> response(200)
199
-
200
- pages = unzip_and_parse_csv(zip_content, ~c"pages.csv")
201
- # header + 100 data rows + trailing empty row
202
- assert length(pages) == 102
203
-
204
- exit_pages = unzip_and_parse_csv(zip_content, ~c"exit_pages.csv")
205
- assert length(exit_pages) == 102
206
- end
207
-
208
- test "limits custom_props.csv to 25 prop keys", %{conn: conn, site: site} do
209
- keys = for i <- 1..26, do: "key_#{i}"
210
- {:ok, site} = Plausible.Props.allow(site, keys)
211
-
212
- populate_stats(site, [
213
- build(:pageview,
214
- "meta.key": keys,
215
- "meta.value": List.duplicate("v", 26)
216
- )
217
- ])
218
-
219
- custom_props =
220
- conn
221
- |> do_export(site, %{@base_params | date_range: "day"})
222
- |> response(200)
223
- |> unzip_and_parse_csv(~c"custom_props.csv")
224
-
225
- prop_keys =
226
- custom_props
227
- |> Enum.drop(1)
228
- |> Enum.reject(&(&1 == [""]))
229
- |> Enum.map(&hd/1)
230
- |> Enum.uniq()
231
-
232
- assert length(prop_keys) == 25
233
- end
234
-
235
- test "limits other breakdown reports to 300 rows", %{conn: conn, site: site} do
236
- events = for i <- 1..301, do: build(:pageview, referrer_source: "Source #{i}")
237
- populate_stats(site, events)
238
-
239
- sources =
240
- conn
241
- |> do_export(site, %{@base_params | date_range: "day"})
242
- |> response(200)
243
- |> unzip_and_parse_csv(~c"sources.csv")
244
-
245
- # header + 300 data rows + trailing empty row
246
- assert length(sources) == 302
247
- end
248
-
249
- test "exports scroll depth metric in pages.csv", %{conn: conn, site: site} do
250
- t0 = ~N[2020-01-01 00:00:00]
251
- [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)
252
-
253
- populate_stats(site, [
254
- build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),
255
- build(:engagement,
256
- user_id: 12,
257
- pathname: "/blog",
258
- timestamp: t1,
259
- scroll_depth: 20,
260
- engagement_time: 60_000
261
- ),
262
- build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),
263
- build(:engagement,
264
- user_id: 12,
265
- pathname: "/another",
266
- timestamp: t2,
267
- scroll_depth: 24,
268
- engagement_time: 60_000
269
- ),
270
- build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),
271
- build(:engagement,
272
- user_id: 34,
273
- pathname: "/blog",
274
- timestamp: t1,
275
- scroll_depth: 17,
276
- engagement_time: 60_000
277
- ),
278
- build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),
279
- build(:engagement,
280
- user_id: 34,
281
- pathname: "/another",
282
- timestamp: t2,
283
- scroll_depth: 26,
284
- engagement_time: 60_000
285
- ),
286
- build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),
287
- build(:engagement,
288
- user_id: 34,
289
- pathname: "/blog",
290
- timestamp: t3,
291
- scroll_depth: 60,
292
- engagement_time: 60_000
293
- ),
294
- build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),
295
- build(:engagement,
296
- user_id: 56,
297
- pathname: "/blog",
298
- timestamp: t1,
299
- scroll_depth: 100,
300
- engagement_time: 60_000
301
- )
302
- ])
303
-
304
- pages =
305
- conn
306
- |> do_export(site, %{@base_params | date_range: "day", relative_date: "2020-01-01"})
307
- |> response(200)
308
- |> unzip_and_parse_csv(~c"pages.csv")
309
-
310
- assert pages == [
311
- ["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],
312
- ["/blog", "3", "4", "33", "80", "60"],
313
- ["/another", "2", "2", "0", "60", "25"],
314
- [""]
315
- ]
316
- end
317
-
318
- test "exports only internally used props in custom_props.csv for a growth plan", %{
319
- conn: conn,
320
- site: site
321
- } do
322
- {:ok, site} = Plausible.Props.allow(site, ["author"])
323
-
324
- [owner | _] = Repo.preload(site, :owners).owners
325
- subscribe_to_growth_plan(owner)
326
-
327
- populate_stats(site, [
328
- build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),
329
- build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])
330
- ])
331
-
332
- result =
333
- conn
334
- |> do_export(site, %{@base_params | date_range: "day"})
335
- |> response(200)
336
- |> unzip_and_parse_csv(~c"custom_props.csv")
337
-
338
- assert result == [
339
- ["property", "value", "visitors", "events", "percentage"],
340
- ["url", "(none)", "1", "1", "50.0"],
341
- ["url", "b", "1", "1", "50.0"],
342
- [""]
343
- ]
344
- end
345
-
346
- test "does not include custom_props.csv for a growth plan if no internal props used", %{
347
- conn: conn,
348
- site: site
349
- } do
350
- {:ok, site} = Plausible.Props.allow(site, ["author"])
351
-
352
- [owner | _] = Repo.preload(site, :owners).owners
353
- subscribe_to_growth_plan(owner)
354
-
355
- populate_stats(site, [
356
- build(:pageview, "meta.key": ["author"], "meta.value": ["a"])
357
- ])
358
-
359
- {:ok, zip} =
360
- conn
361
- |> do_export(site, %{@base_params | date_range: "day"})
362
- |> response(200)
363
- |> :zip.unzip([:memory])
364
-
365
- files = Map.new(zip)
366
-
367
- refute Map.has_key?(files, ~c"custom_props.csv")
368
- end
369
-
370
- test "exports data in zipped csvs", %{conn: conn, site: site} do
371
- populate_exported_stats(site)
372
-
373
- conn =
374
- do_export(conn, site, %{@base_params | date_range: ["2021-09-20", "2021-10-20"]})
375
-
376
- assert_zip(conn, "30d")
377
- end
378
-
379
- test "exports allowed event props for a trial account", %{conn: conn, site: site} do
380
- {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
381
-
382
- populate_stats(site, [
383
- build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
384
- build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
385
- build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),
386
- build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
387
- build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
388
- build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),
389
- build(:pageview)
390
- ])
391
-
392
- result =
393
- conn
394
- |> do_export(site, %{@base_params | date_range: "day"})
395
- |> response(200)
396
- |> unzip_and_parse_csv(~c"custom_props.csv")
397
-
398
- assert result == [
399
- ["property", "value", "visitors", "events", "percentage"],
400
- ["author", "(none)", "3", "4", "50.0"],
401
- ["author", "uku", "2", "2", "33.33"],
402
- ["author", "marko", "1", "1", "16.67"],
403
- ["logged_in", "(none)", "5", "5", "83.33"],
404
- ["logged_in", "true", "1", "2", "16.67"],
405
- [""]
406
- ]
407
- end
408
-
409
- test "exports data grouped by interval", %{conn: conn, site: site} do
410
- populate_exported_stats(site)
411
-
412
- params =
413
- @base_params
414
- |> Map.put(:date_range, ["2021-09-20", "2021-10-20"])
415
- |> put_in([:reports, "visitors.csv", :dimensions], ["time:week"])
416
-
417
- visitors =
418
- conn
419
- |> do_export(site, params)
420
- |> response(200)
421
- |> unzip_and_parse_csv(~c"visitors.csv")
422
-
423
- assert visitors == [
424
- [
425
- "date",
426
- "visitors",
427
- "pageviews",
428
- "visits",
429
- "views_per_visit",
430
- "bounce_rate",
431
- "visit_duration"
432
- ],
433
- ["2021-09-20", "1", "1", "1", "1.0", "100", "0"],
434
- ["2021-09-27", "0", "0", "0", "0.0", "0.0", ""],
435
- ["2021-10-04", "0", "0", "0", "0.0", "0.0", ""],
436
- ["2021-10-11", "0", "0", "0", "0.0", "0.0", ""],
437
- ["2021-10-18", "3", "4", "3", "1.33", "33", "40"],
438
- [""]
439
- ]
440
- end
441
-
442
- test "exports operating system versions", %{conn: conn, site: site} do
443
- populate_stats(site, [
444
- build(:pageview, operating_system: "Mac", operating_system_version: "14"),
445
- build(:pageview, operating_system: "Mac", operating_system_version: "14"),
446
- build(:pageview, operating_system: "Mac", operating_system_version: "14"),
447
- build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),
448
- build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),
449
- build(:pageview, operating_system: "Mac", operating_system_version: "13")
450
- ])
451
-
452
- os_versions =
453
- conn
454
- |> do_export(site, %{@base_params | date_range: "day"})
455
- |> response(200)
456
- |> unzip_and_parse_csv(~c"operating_system_versions.csv")
457
-
458
- assert os_versions == [
459
- ["name", "version", "visitors"],
460
- ["Mac", "14", "3"],
461
- ["Ubuntu", "20.04", "2"],
462
- ["Mac", "13", "1"],
463
- [""]
464
- ]
465
- end
466
-
467
- test "exports imported data when requested", %{conn: conn, site: site} do
468
- site_import = insert(:site_import, site: site)
469
-
470
- insert(:goal, site: site, event_name: "Outbound Link: Click")
471
-
472
- populate_stats(site, site_import.id, [
473
- build(:imported_visitors, visitors: 9),
474
- build(:imported_browsers, browser: "Chrome", pageviews: 1),
475
- build(:imported_devices, device: "Desktop", pageviews: 1),
476
- build(:imported_entry_pages, entry_page: "/test", pageviews: 1),
477
- build(:imported_exit_pages, exit_page: "/test", pageviews: 1),
478
- build(:imported_locations, country: "PL", region: "PL-22", city: 3_099_434, pageviews: 1),
479
- build(:imported_operating_systems, operating_system: "Mac", pageviews: 1),
480
- build(:imported_pages, page: "/test", pageviews: 1),
481
- build(:imported_sources,
482
- source: "Google",
483
- channel: "Paid Search",
484
- utm_medium: "search",
485
- utm_campaign: "ads",
486
- utm_source: "google",
487
- utm_content: "content",
488
- utm_term: "term",
489
- pageviews: 1
490
- ),
491
- build(:imported_custom_events,
492
- name: "Outbound Link: Click",
493
- link_url: "https://example.com",
494
- visitors: 5,
495
- events: 10
496
- )
497
- ])
498
-
499
- tomorrow = Date.utc_today() |> Date.add(1) |> Date.to_iso8601()
500
-
501
- params = %{@base_params | relative_date: tomorrow, include: %{imports: true}}
502
-
503
- assert response = do_export(conn, site, params) |> response(200)
504
- {:ok, zip} = :zip.unzip(response, [:memory])
505
-
506
- filenames = zip |> Enum.map(fn {filename, _} -> to_string(filename) end)
507
-
508
- # NOTE: currently, custom_props.csv is not populated from imported data
509
- expected_filenames = [
510
- "visitors.csv",
511
- "sources.csv",
512
- "channels.csv",
513
- "utm_mediums.csv",
514
- "utm_sources.csv",
515
- "utm_campaigns.csv",
516
- "utm_contents.csv",
517
- "utm_terms.csv",
518
- "pages.csv",
519
- "entry_pages.csv",
520
- "exit_pages.csv",
521
- "countries.csv",
522
- "regions.csv",
523
- "cities.csv",
524
- "browsers.csv",
525
- "browser_versions.csv",
526
- "operating_systems.csv",
527
- "operating_system_versions.csv",
528
- "devices.csv",
529
- "conversions.csv",
530
- "referrers.csv"
531
- ]
532
-
533
- Enum.each(expected_filenames, fn expected -> assert expected in filenames end)
534
-
535
- Enum.each(zip, fn
536
- {~c"visitors.csv", data} ->
537
- csv = parse_csv(data)
538
-
539
- assert List.first(csv) == [
540
- "date",
541
- "visitors",
542
- "pageviews",
543
- "visits",
544
- "views_per_visit",
545
- "bounce_rate",
546
- "visit_duration"
547
- ]
548
-
549
- assert Enum.at(csv, -2) ==
550
- [Date.to_iso8601(Date.utc_today()), "9", "1", "1", "1.0", "0.0", "10.0"]
551
-
552
- {~c"sources.csv", data} ->
553
- assert parse_csv(data) == [
554
- ["name", "visitors", "bounce_rate", "visit_duration"],
555
- ["Google", "1", "0.0", "10.0"],
556
- [""]
557
- ]
558
-
559
- {~c"channels.csv", data} ->
560
- assert parse_csv(data) == [
561
- ["name", "visitors", "bounce_rate", "visit_duration"],
562
- ["Paid Search", "1", "0.0", "10.0"],
563
- [""]
564
- ]
565
-
566
- {~c"utm_mediums.csv", data} ->
567
- assert parse_csv(data) == [
568
- ["name", "visitors", "bounce_rate", "visit_duration"],
569
- ["search", "1", "0.0", "10.0"],
570
- [""]
571
- ]
572
-
573
- {~c"utm_sources.csv", data} ->
574
- assert parse_csv(data) == [
575
- ["name", "visitors", "bounce_rate", "visit_duration"],
576
- ["google", "1", "0.0", "10.0"],
577
- [""]
578
- ]
579
-
580
- {~c"utm_campaigns.csv", data} ->
581
- assert parse_csv(data) == [
582
- ["name", "visitors", "bounce_rate", "visit_duration"],
583
- ["ads", "1", "0.0", "10.0"],
584
- [""]
585
- ]
586
-
587
- {~c"utm_contents.csv", data} ->
588
- assert parse_csv(data) == [
589
- ["name", "visitors", "bounce_rate", "visit_duration"],
590
- ["content", "1", "0.0", "10.0"],
591
- [""]
592
- ]
593
-
594
- {~c"utm_terms.csv", data} ->
595
- assert parse_csv(data) == [
596
- ["name", "visitors", "bounce_rate", "visit_duration"],
597
- ["term", "1", "0.0", "10.0"],
598
- [""]
599
- ]
600
-
601
- {~c"pages.csv", data} ->
602
- assert parse_csv(data) == [
603
- [
604
- "name",
605
- "visitors",
606
- "pageviews",
607
- "bounce_rate",
608
- "time_on_page",
609
- "scroll_depth"
610
- ],
611
- ["/test", "1", "1", "0.0", "10", ""],
612
- [""]
613
- ]
614
-
615
- {~c"entry_pages.csv", data} ->
616
- assert parse_csv(data) == [
617
- [
618
- "name",
619
- "unique_entrances",
620
- "total_entrances",
621
- "bounce_rate",
622
- "visit_duration"
623
- ],
624
- ["/test", "1", "1", "0.0", "10.0"],
625
- [""]
626
- ]
627
-
628
- {~c"exit_pages.csv", data} ->
629
- assert parse_csv(data) == [
630
- ["name", "unique_exits", "total_exits", "exit_rate"],
631
- ["/test", "1", "1", "100.0"],
632
- [""]
633
- ]
634
-
635
- {~c"countries.csv", data} ->
636
- assert parse_csv(data) == [["name", "visitors"], ["Poland", "1"], [""]]
637
-
638
- {~c"regions.csv", data} ->
639
- assert parse_csv(data) == [["name", "visitors"], ["Pomerania", "1"], [""]]
640
-
641
- {~c"cities.csv", data} ->
642
- assert parse_csv(data) == [["name", "visitors"], ["Gdańsk", "1"], [""]]
643
-
644
- {~c"browsers.csv", data} ->
645
- assert parse_csv(data) == [["name", "visitors"], ["Chrome", "1"], [""]]
646
-
647
- {~c"browser_versions.csv", data} ->
648
- assert parse_csv(data) == [
649
- ["name", "version", "visitors"],
650
- ["Chrome", "(not set)", "1"],
651
- [""]
652
- ]
653
-
654
- {~c"operating_systems.csv", data} ->
655
- assert parse_csv(data) == [["name", "visitors"], ["Mac", "1"], [""]]
656
-
657
- {~c"operating_system_versions.csv", data} ->
658
- assert parse_csv(data) == [
659
- ["name", "version", "visitors"],
660
- ["Mac", "(not set)", "1"],
661
- [""]
662
- ]
663
-
664
- {~c"devices.csv", data} ->
665
- assert parse_csv(data) == [["name", "visitors"], ["Desktop", "1"], [""]]
666
-
667
- {~c"conversions.csv", data} ->
668
- assert parse_csv(data) == [
669
- ["name", "unique_conversions", "total_conversions"],
670
- ["Outbound Link: Click", "5", "10"],
671
- [""]
672
- ]
673
-
674
- {~c"referrers.csv", data} ->
675
- assert parse_csv(data) == [
676
- ["name", "visitors", "bounce_rate", "visit_duration"],
677
- ["Direct / None", "1", "0.0", "10.0"],
678
- [""]
679
- ]
680
-
681
- {~c"custom_props.csv", _data} ->
682
- :ok
683
- end)
684
- end
685
- end
686
-
687
- defp parse_csv(file_content) when is_binary(file_content) do
688
- file_content
689
- |> String.split("\r\n")
690
- |> Enum.map(&String.split(&1, ","))
691
- end
692
-
693
- describe "POST /api/stats/:domain/export - via shared link" do
694
- setup [:create_user, :create_site]
695
-
696
- test "exports data in zipped csvs", %{conn: conn, site: site} do
697
- link = insert(:shared_link, site: site)
698
-
699
- populate_exported_stats(site)
700
-
701
- params =
702
- %{@base_params | date_range: ["2021-09-20", "2021-10-20"], include: %{imports: false}}
703
- |> Map.put(:auth, link.slug)
704
-
705
- conn = do_export(conn, site, params)
706
-
707
- assert_zip(conn, "30d")
708
- end
709
- end
710
-
711
- describe "POST /api/stats/:domain/export - for past 6 months" do
712
- setup [:create_user, :create_site, :log_in]
713
-
714
- test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
715
- populate_exported_stats(site)
716
-
717
- conn =
718
- do_export(conn, site, %{
719
- @base_params
720
- | date_range: "6mo",
721
- relative_date: "2021-11-20",
722
- reports: put_in(@default_reports, ["visitors.csv", :dimensions], ["time:month"])
723
- })
724
-
725
- assert_zip(conn, "6m")
726
- end
727
- end
728
-
729
- describe "POST /api/stats/:domain/export - with path filter" do
730
- setup [:create_user, :create_site, :log_in]
731
-
732
- test "exports filtered data in zipped csvs", %{conn: conn, site: site} do
733
- populate_exported_stats(site)
734
-
735
- conn =
736
- do_export(conn, site, %{
737
- @base_params
738
- | date_range: ["2021-09-20", "2021-10-20"],
739
- filters: [["is", "event:page", ["/some-other-page"]]],
740
- reports: @page_filtered_reports
741
- })
742
-
743
- assert_zip(conn, "30d-filter-path")
744
- end
745
-
746
- test "exports scroll depth in visitors.csv", %{conn: conn, site: site} do
747
- populate_stats(site, [
748
- build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 00:00:00]),
749
- build(:engagement,
750
- user_id: 12,
751
- pathname: "/blog",
752
- timestamp: ~N[2020-01-05 00:01:00],
753
- scroll_depth: 40
754
- ),
755
- build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 10:00:00]),
756
- build(:engagement,
757
- user_id: 12,
758
- pathname: "/blog",
759
- timestamp: ~N[2020-01-05 10:01:00],
760
- scroll_depth: 17
761
- ),
762
- build(:pageview, user_id: 34, pathname: "/blog", timestamp: ~N[2020-01-07 00:00:00]),
763
- build(:engagement,
764
- user_id: 34,
765
- pathname: "/blog",
766
- timestamp: ~N[2020-01-07 00:01:00],
767
- scroll_depth: 90
768
- )
769
- ])
770
-
771
- pages =
772
- conn
773
- |> do_export(site, %{
774
- @base_params
775
- | date_range: "7d",
776
- relative_date: "2020-01-08",
777
- filters: [["is", "event:page", ["/blog"]]],
778
- reports: @page_filtered_reports
779
- })
780
- |> response(200)
781
- |> unzip_and_parse_csv(~c"visitors.csv")
782
-
783
- assert pages == [
784
- [
785
- "date",
786
- "visitors",
787
- "pageviews",
788
- "visits",
789
- "bounce_rate",
790
- "time_on_page",
791
- "scroll_depth"
792
- ],
793
- ["2020-01-01", "0", "0", "0", "0.0", "", ""],
794
- ["2020-01-02", "0", "0", "0", "0.0", "", ""],
795
- ["2020-01-03", "0", "0", "0", "0.0", "", ""],
796
- ["2020-01-04", "0", "0", "0", "0.0", "", ""],
797
- ["2020-01-05", "1", "2", "2", "100", "0", "28"],
798
- ["2020-01-06", "0", "0", "0", "0.0", "", ""],
799
- ["2020-01-07", "1", "1", "1", "100", "0", "90"],
800
- [""]
801
- ]
802
- end
803
- end
804
-
805
- describe "POST /api/stats/:domain/export - with a custom prop filter" do
806
- setup [:create_user, :create_site, :log_in]
807
-
808
- test "custom-props.csv only returns the prop and its value in filter", %{
809
- conn: conn,
810
- site: site
811
- } do
812
- {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
813
-
814
- populate_stats(site, [
815
- build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
816
- build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
817
- build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
818
- ])
819
-
820
- result =
821
- conn
822
- |> do_export(site, %{
823
- @base_params
824
- | date_range: "day",
825
- filters: [["is", "event:props:author", ["marko"]]],
826
- reports: @prop_filtered_reports
827
- })
828
- |> response(200)
829
- |> unzip_and_parse_csv(~c"custom_props.csv")
830
-
831
- assert result == [
832
- ["property", "value", "visitors", "events", "percentage"],
833
- ["author", "marko", "1", "1", "100.0"],
834
- [""]
835
- ]
836
- end
837
- end
838
-
839
- defp unzip_and_parse_csv(archive, filename) do
840
- {:ok, zip} = :zip.unzip(archive, [:memory])
841
- {_filename, data} = Enum.find(zip, &(elem(&1, 0) == filename))
842
- parse_csv(data)
843
- end
844
-
845
- defp assert_zip(conn, folder) do
846
- assert conn.status == 200
847
-
848
- assert {"content-type", "application/zip; charset=utf-8"} =
849
- List.keyfind(conn.resp_headers, "content-type", 0)
850
-
851
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
852
-
853
- folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")
854
-
855
- Enum.map(zip, &assert_csv_by_fixture(&1, folder))
856
- end
857
-
858
- defp assert_csv_by_fixture({file, downloaded}, folder) do
859
- file = Path.expand(file, folder)
860
-
861
- {:ok, content} = File.read(file)
862
- msg = "CSV file comparison failed (#{file})"
863
- assert downloaded == content, message: msg, left: downloaded, right: content
864
- end
865
-
866
- defp do_export(conn, site, params) do
867
- conn
868
- |> put_req_header("content-type", "application/json")
869
- |> post("/api/stats/#{site.domain}/export", Jason.encode!(params))
870
- end
871
-
872
- defp populate_exported_stats(site) do
873
- populate_stats(site, [
874
- build(:pageview,
875
- user_id: 123,
876
- pathname: "/",
877
- timestamp:
878
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)
879
- |> NaiveDateTime.truncate(:second),
880
- country_code: "EE",
881
- subdivision1_code: "EE-37",
882
- city_geoname_id: 588_409,
883
- referrer_source: "Google"
884
- ),
885
- build(:engagement,
886
- user_id: 123,
887
- pathname: "/",
888
- timestamp: ~N[2021-10-20 12:00:00] |> NaiveDateTime.truncate(:second),
889
- engagement_time: 30_000,
890
- scroll_depth: 30,
891
- country_code: "EE",
892
- subdivision1_code: "EE-37",
893
- city_geoname_id: 588_409,
894
- referrer_source: "Google"
895
- ),
896
- build(:pageview,
897
- user_id: 123,
898
- pathname: "/some-other-page",
899
- timestamp:
900
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -2)
901
- |> NaiveDateTime.truncate(:second),
902
- country_code: "EE",
903
- subdivision1_code: "EE-37",
904
- city_geoname_id: 588_409,
905
- referrer_source: "Google"
906
- ),
907
- build(:engagement,
908
- user_id: 123,
909
- pathname: "/some-other-page",
910
- timestamp:
911
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)
912
- |> NaiveDateTime.truncate(:second),
913
- engagement_time: 60_000,
914
- scroll_depth: 30,
915
- country_code: "EE",
916
- subdivision1_code: "EE-37",
917
- city_geoname_id: 588_409,
918
- referrer_source: "Google"
919
- ),
920
- build(:pageview,
921
- user_id: 100,
922
- pathname: "/",
923
- timestamp:
924
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
925
- utm_medium: "search",
926
- utm_campaign: "ads",
927
- utm_source: "google",
928
- utm_content: "content",
929
- utm_term: "term",
930
- browser: "Firefox",
931
- browser_version: "120",
932
- operating_system: "Mac",
933
- operating_system_version: "14"
934
- ),
935
- build(:engagement,
936
- user_id: 100,
937
- pathname: "/",
938
- timestamp:
939
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: 1)
940
- |> NaiveDateTime.truncate(:second),
941
- engagement_time: 30_000,
942
- scroll_depth: 30,
943
- utm_medium: "search",
944
- utm_campaign: "ads",
945
- utm_source: "google",
946
- utm_content: "content",
947
- utm_term: "term",
948
- browser: "Firefox",
949
- browser_version: "120",
950
- operating_system: "Mac",
951
- operating_system_version: "14"
952
- ),
953
- build(:pageview,
954
- user_id: 200,
955
- timestamp:
956
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1)
957
- |> NaiveDateTime.truncate(:second),
958
- country_code: "EE",
959
- browser: "Firefox",
960
- browser_version: "120",
961
- operating_system: "Mac",
962
- operating_system_version: "14"
963
- ),
964
- build(:engagement,
965
- user_id: 200,
966
- timestamp:
967
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1, minute: 1)
968
- |> NaiveDateTime.truncate(:second),
969
- engagement_time: 30_000,
970
- scroll_depth: 20,
971
- country_code: "EE",
972
- browser: "Firefox",
973
- browser_version: "120",
974
- operating_system: "Mac",
975
- operating_system_version: "14"
976
- ),
977
- build(:pageview,
978
- user_id: 300,
979
- timestamp:
980
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5)
981
- |> NaiveDateTime.truncate(:second),
982
- utm_campaign: "ads",
983
- country_code: "EE",
984
- referrer_source: "Google",
985
- click_id_param: "gclid",
986
- browser: "FirefoxNoVersion",
987
- operating_system: "MacNoVersion"
988
- ),
989
- build(:engagement,
990
- user_id: 300,
991
- timestamp:
992
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5, minute: 1)
993
- |> NaiveDateTime.truncate(:second),
994
- engagement_time: 30_000,
995
- scroll_depth: 20,
996
- utm_campaign: "ads",
997
- country_code: "EE",
998
- referrer_source: "Google",
999
- click_id_param: "gclid",
1000
- browser: "FirefoxNoVersion",
1001
- operating_system: "MacNoVersion"
1002
- ),
1003
- build(:pageview,
1004
- user_id: 456,
1005
- timestamp:
1006
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: -1)
1007
- |> NaiveDateTime.truncate(:second),
1008
- pathname: "/signup",
1009
- "meta.key": ["variant"],
1010
- "meta.value": ["A"]
1011
- ),
1012
- build(:engagement,
1013
- user_id: 456,
1014
- timestamp:
1015
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
1016
- pathname: "/signup",
1017
- engagement_time: 60_000,
1018
- scroll_depth: 20,
1019
- "meta.key": ["variant"],
1020
- "meta.value": ["A"]
1021
- ),
1022
- build(:event,
1023
- user_id: 456,
1024
- timestamp:
1025
- NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
1026
- name: "Signup",
1027
- "meta.key": ["variant"],
1028
- "meta.value": ["A"]
1029
- )
1030
- ])
1031
-
1032
- insert(:goal, %{site: site, event_name: "Signup"})
1033
- end
1034
-
1035
- describe "POST /api/stats/:domain/export - with goal filter" do
1036
- setup [:create_user, :create_site, :log_in]
1037
-
1038
- test "exports goal-filtered data in zipped csvs", %{conn: conn, site: site} do
1039
- populate_exported_stats(site)
1040
-
1041
- conn =
1042
- do_export(conn, site, %{
1043
- @base_params
1044
- | date_range: ["2021-09-20", "2021-10-20"],
1045
- filters: [["is", "event:goal", ["Signup"]]],
1046
- reports: @goal_filtered_reports
1047
- })
1048
-
1049
- assert_zip(conn, "30d-filter-goal")
1050
- end
1051
-
1052
- test "custom-props.csv only returns the prop names for the goal in filter", %{
1053
- conn: conn,
1054
- site: site
1055
- } do
1056
- {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
1057
-
1058
- populate_stats(site, [
1059
- build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),
1060
- build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
1061
- build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
1062
- build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
1063
- ])
1064
-
1065
- insert(:goal, site: site, event_name: "Newsletter Signup")
1066
-
1067
- result =
1068
- conn
1069
- |> do_export(site, %{
1070
- @base_params
1071
- | date_range: "day",
1072
- filters: [["is", "event:goal", ["Newsletter Signup"]]],
1073
- reports: @goal_filtered_reports
1074
- })
1075
- |> response(200)
1076
- |> unzip_and_parse_csv(~c"custom_props.csv")
1077
-
1078
- assert result == [
1079
- ["property", "value", "visitors", "events", "conversion_rate"],
1080
- ["author", "marko", "2", "2", "50.0"],
1081
- ["author", "uku", "1", "1", "25.0"],
1082
- [""]
1083
- ]
1084
- end
1085
-
1086
- test "exports conversions and conversion rate for operating system versions", %{
1087
- conn: conn,
1088
- site: site
1089
- } do
1090
- populate_stats(site, [
1091
- build(:pageview, operating_system: "Mac", operating_system_version: "14"),
1092
- build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
1093
- build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
1094
- build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
1095
- build(:event,
1096
- name: "Signup",
1097
- operating_system: "Ubuntu",
1098
- operating_system_version: "20.04"
1099
- ),
1100
- build(:event,
1101
- name: "Signup",
1102
- operating_system: "Ubuntu",
1103
- operating_system_version: "20.04"
1104
- ),
1105
- build(:event,
1106
- name: "Signup",
1107
- operating_system: "Lubuntu",
1108
- operating_system_version: "20.04"
1109
- )
1110
- ])
1111
-
1112
- insert(:goal, site: site, event_name: "Signup")
1113
-
1114
- os_versions =
1115
- conn
1116
- |> do_export(site, %{
1117
- @base_params
1118
- | date_range: "day",
1119
- filters: [["is", "event:goal", ["Signup"]]],
1120
- reports: @goal_filtered_reports
1121
- })
1122
- |> response(200)
1123
- |> unzip_and_parse_csv(~c"operating_system_versions.csv")
1124
-
1125
- assert os_versions == [
1126
- ["name", "version", "conversions", "conversion_rate"],
1127
- ["Mac", "14", "3", "75.0"],
1128
- ["Ubuntu", "20.04", "2", "100.0"],
1129
- ["Lubuntu", "20.04", "1", "100.0"],
1130
- [""]
1131
- ]
1132
- end
1133
- end
1134
- end