Skip to content

Commit 89ed9cd

Browse files
ptitfredptitfred
committed
NeTEx: meilleure sérialisation des résultats
- [x] Ajoute une colonne `multi_validation.binary_result` (bytea). - [x] Script de backfill de cette colonne. - [x] Remplit le `binary_result` à la validation. - [x] Ne stocke plus la première page des issues dans le `digest`. - [x] Page de détails d’une ressource NeTEx : utilise le `digest` et le `binary_result` à la place du `result`. - [x] Page de validation à la demande NeTEx : utilise le `digest` et le `binary_result` à la place du `result`.
1 parent 83b7320 commit 89ed9cd

File tree

19 files changed

+532
-74
lines changed

19 files changed

+532
-74
lines changed

‎apps/transport/lib/db/multi_validation.ex‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,26 @@ defmodule DB.MultiValidation do
3131
timestamps(type: :utc_datetime_usec)
3232

3333
field(:digest, :map)
34+
field(:binary_result, :binary, load_in_query: false)
3435
end
3536

3637
def base_query(opts \\ []) do
3738
include_result = Keyword.get(opts, :include_result, false)
39+
include_binary_result = Keyword.get(opts, :include_binary_result, false)
3840

39-
if include_result do
40-
from(mv in DB.MultiValidation, as: :multi_validation)
41-
|> select_merge([mv], %{result: mv.result})
41+
query = from(mv in DB.MultiValidation, as: :multi_validation)
42+
43+
query =
44+
if include_result do
45+
from(mv in query, select_merge: %{result: mv.result})
46+
else
47+
query
48+
end
49+
50+
if include_binary_result do
51+
from(mv in query, select_merge: %{binary_result: mv.binary_result})
4252
else
43-
from(mv in DB.MultiValidation, as: :multi_validation)
53+
query
4454
end
4555
end
4656

‎apps/transport/lib/transport_web/controllers/resource_controller.ex‎

Lines changed: 58 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -158,15 +158,22 @@ defmodule TransportWeb.ResourceController do
158158
Enum.empty?(validators) -> nil
159159
end
160160

161-
DB.MultiValidation.resource_latest_validation(resource_id, validator, include_result: true)
161+
netex? = validator == Transport.Validators.NeTEx.Validator
162+
163+
DB.MultiValidation.resource_latest_validation(resource_id, validator,
164+
include_result: not netex?,
165+
include_binary_result: netex?
166+
)
162167
end
163168

164169
def render_details(conn, resource) do
165170
conn |> assign(:resource, resource) |> render("details.html")
166171
end
167172

168173
defp render_gtfs_details(conn, params, resource, validation) do
169-
validation_details = {_, _, _, _, issues} = build_gtfs_validation_details(validation, params)
174+
config = make_pagination_config(params)
175+
176+
{validation_details, issues} = build_gtfs_validation_details(validation, params)
170177

171178
issue_type =
172179
case params["issue_type"] do
@@ -175,65 +182,97 @@ defmodule TransportWeb.ResourceController do
175182
end
176183

177184
conn
178-
|> assign_base_resource_details(params, resource, validation_details)
185+
|> assign_base_resource_details(resource, validation_details)
186+
|> assign(:issues, Scrivener.paginate(issues, config))
179187
|> assign(:validator, Transport.Validators.GTFSTransport)
180188
|> assign(:data_vis, encoded_data_vis(issue_type, validation))
181189
|> render("gtfs_details.html")
182190
end
183191

184-
defp build_gtfs_validation_details(nil, _params), do: {nil, nil, nil, [], []}
192+
defp build_gtfs_validation_details(nil, _params), do: {{nil, nil, nil, []}, []}
185193

186194
defp build_gtfs_validation_details(%{result: validation_result, metadata: metadata = %DB.ResourceMetadata{}}, params) do
187195
summary = Transport.Validators.GTFSTransport.summary(validation_result)
188196
stats = Transport.Validators.GTFSTransport.count_by_severity(validation_result)
189197
issues = Transport.Validators.GTFSTransport.get_issues(validation_result, params)
190198

191-
{summary, stats, metadata.metadata, metadata.modes, issues}
199+
{{summary, stats, metadata.metadata, metadata.modes}, issues}
192200
end
193201

194202
defp render_netex_details(conn, params, resource, validation) do
195-
{results_adapter, validation_details, errors_template, max_severity} =
203+
config = make_pagination_config(params)
204+
205+
{results_adapter, validation_details, issues, errors_template, max_severity} =
196206
build_netex_validation_details(validation, params)
197207

208+
{filter, pagination} = issues
209+
198210
conn
199-
|> assign_base_resource_details(params, resource, validation_details)
211+
|> assign_base_resource_details(resource, validation_details)
212+
|> assign(:filter, filter)
213+
|> assign(:issues, paginate_netex_results(pagination, config))
200214
|> assign(:errors_template, errors_template)
201215
|> assign(:results_adapter, results_adapter)
202216
|> assign(:max_severity, max_severity)
203217
|> assign(:data_vis, nil)
204218
|> render("netex_details.html")
205219
end
206220

207-
defp build_netex_validation_details(nil, _params), do: {nil, {nil, nil, nil, [], []}, nil, nil}
221+
# For NeTEx results we avoid loading every entries. We emulate
222+
# Scrivener.paginate based on the total count.
223+
def paginate_netex_results({total_entries, issues}, config) do
224+
total_pages = div(total_entries, config.page_size)
225+
226+
total_pages =
227+
if rem(total_entries, config.page_size) > 0 do
228+
total_pages + 1
229+
else
230+
total_pages
231+
end
232+
233+
%Scrivener.Page{
234+
entries: issues,
235+
page_number: config.page_number,
236+
page_size: config.page_size,
237+
total_entries: total_entries,
238+
total_pages: total_pages
239+
}
240+
end
241+
242+
defp build_netex_validation_details(nil, _params), do: {nil, {nil, nil, nil, []}, {%{}, {0, []}}, nil, nil}
208243

209244
defp build_netex_validation_details(
210-
%{validator_version: version, result: validation_result, metadata: metadata = %DB.ResourceMetadata{}},
245+
%{
246+
validator_version: version,
247+
digest: digest,
248+
binary_result: binary_result,
249+
metadata: metadata = %DB.ResourceMetadata{}
250+
},
211251
params
212252
) do
213253
results_adapter = Transport.Validators.NeTEx.ResultsAdapter.resolve(version)
214-
summary = results_adapter.summary(validation_result)
215-
stats = results_adapter.count_by_severity(validation_result)
216-
issues = results_adapter.get_issues(validation_result, params)
254+
summary = digest["summary"]
255+
stats = digest["stats"]
217256
errors_template = pick_netex_errors_template(version)
218-
max_severity = results_adapter.count_max_severity(validation_result)
257+
max_severity = digest["max_severity"]
219258

220-
{results_adapter, {summary, stats, metadata.metadata, metadata.modes, issues}, errors_template, max_severity}
259+
pagination_config = make_pagination_config(params)
260+
issues = results_adapter.get_issues(binary_result, params, pagination_config)
261+
262+
{results_adapter, {summary, stats, metadata.metadata, metadata.modes}, issues, errors_template, max_severity}
221263
end
222264

223265
defp pick_netex_errors_template("0.2.1"), do: "_netex_validation_errors_v0_2_x.html"
224266
defp pick_netex_errors_template("0.2.0"), do: "_netex_validation_errors_v0_2_x.html"
225267
defp pick_netex_errors_template(_), do: "_netex_validation_errors_v0_1_0.html"
226268

227-
defp assign_base_resource_details(conn, params, resource, validation_details) do
228-
config = make_pagination_config(params)
229-
230-
{validation_summary, severities_count, metadata, modes, issues} = validation_details
269+
defp assign_base_resource_details(conn, resource, validation_details) do
270+
{validation_summary, severities_count, metadata, modes} = validation_details
231271

232272
conn
233273
|> assign(:related_files, Resource.get_related_files(resource))
234274
|> assign(:resource, resource)
235275
|> assign(:other_resources, Resource.other_resources(resource))
236-
|> assign(:issues, Scrivener.paginate(issues, config))
237276
|> assign(:validation_summary, validation_summary)
238277
|> assign(:severities_count, severities_count)
239278
|> assign(:metadata, metadata)

‎apps/transport/lib/transport_web/controllers/validation_controller.ex‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,11 @@ defmodule TransportWeb.ValidationController do
8989

9090
def show(%Plug.Conn{} = conn, %{} = params) do
9191
token = params["token"]
92-
validation = MultiValidation.with_result() |> preload(:metadata) |> Repo.get(params["id"])
92+
93+
validation =
94+
MultiValidation.base_query(include_result: true, include_binary_result: true)
95+
|> preload(:metadata)
96+
|> Repo.get(params["id"])
9397

9498
case validation do
9599
nil ->
@@ -99,7 +103,7 @@ defmodule TransportWeb.ValidationController do
99103
when expected_token != token ->
100104
unauthorized(conn)
101105

102-
%MultiValidation{oban_args: %{"state" => "completed"}, result: nil} = validation ->
106+
%MultiValidation{oban_args: %{"state" => "completed"}, binary_result: nil, result: nil} = validation ->
103107
conn |> assign(:validation, validation) |> render("expired.html")
104108

105109
%MultiValidation{oban_args: %{"state" => "completed", "type" => "gtfs"}} = validation ->
@@ -113,25 +117,35 @@ defmodule TransportWeb.ValidationController do
113117
end
114118

115119
conn
116-
|> assign_base_validation_details(validator, validation, params, current_issues)
120+
|> assign_base_validation_details(params)
121+
|> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
117122
|> assign(:validator, validator)
118123
|> assign(:metadata, validation.metadata.metadata)
119124
|> assign(:modes, validation.metadata.modes)
120125
|> assign(:data_vis, data_vis(validation, issue_type))
126+
|> assign(:validation_summary, validator.summary(validation.result))
127+
|> assign(:severities_count, validator.count_by_severity(validation.result))
121128
|> render("show_gtfs.html")
122129

123130
%MultiValidation{oban_args: %{"state" => "completed", "type" => "netex"}} = validation ->
131+
config = make_pagination_config(params)
132+
124133
results_adapter = Transport.Validators.NeTEx.ResultsAdapter.resolve(validation.validator_version)
125134

126135
template = pick_netex_template(validation.validator_version)
127136

128-
current_issues = results_adapter.get_issues(validation.result, params)
137+
pagination_config = make_pagination_config(params)
138+
{filter, pagination} = results_adapter.get_issues(validation.binary_result, params, pagination_config)
129139

130140
conn
131-
|> assign_base_validation_details(results_adapter, validation, params, current_issues)
141+
|> assign_base_validation_details(params)
142+
|> assign(:filter, filter)
143+
|> assign(:issues, TransportWeb.ResourceController.paginate_netex_results(pagination, config))
132144
|> assign(:results_adapter, results_adapter)
133145
|> assign(:metadata, validation.metadata.metadata)
134-
|> assign(:max_severity, results_adapter.count_max_severity(validation.result))
146+
|> assign(:max_severity, validation.digest["max_severity"])
147+
|> assign(:validation_summary, validation.digest["summary"])
148+
|> assign(:severities_count, validation.digest["stats"])
135149
|> render(template)
136150

137151
# Handles waiting for validation to complete, errors and
@@ -151,13 +165,10 @@ defmodule TransportWeb.ValidationController do
151165
defp pick_netex_template("0.2.0"), do: "show_netex_v0_2_x.html"
152166
defp pick_netex_template(_), do: "show_netex_v0_1_0.html"
153167

154-
defp assign_base_validation_details(conn, results_adapter, validation, params, current_issues) do
168+
defp assign_base_validation_details(conn, params) do
155169
conn
156170
|> assign(:validation_id, params["id"])
157171
|> assign(:other_resources, [])
158-
|> assign(:issues, Scrivener.paginate(current_issues, make_pagination_config(params)))
159-
|> assign(:validation_summary, results_adapter.summary(validation.result))
160-
|> assign(:severities_count, results_adapter.count_by_severity(validation.result))
161172
|> assign(:token, params["token"])
162173
end
163174

‎apps/transport/lib/transport_web/templates/resource/_netex_validation_errors_v0_2_x.html.heex‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<% %{"max_level" => max_level, "worst_occurrences" => worst_occurrences} = @max_severity %>
2-
<% current_category = @conn.params["issues_category"] || "" %>
2+
<% current_category = @filter["issues_category"] || "" %>
33

44
<h4><%= @results_adapter.format_severity(max_level, worst_occurrences) |> String.capitalize() %></h4>
55
<.netex_validation_summary

‎apps/transport/lib/transport_web/templates/resource/netex_details.html.heex‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
severities_count: @severities_count,
3030
validation_summary: @validation_summary,
3131
results_adapter: @results_adapter,
32-
max_severity: @max_severity
32+
max_severity: @max_severity,
33+
filter: @filter
3334
) %>
3435
<%= unless is_nil(@metadata) or @metadata == %{} do %>
3536
<%= render("_resources_details_netex.html", conn: @conn, metadata: @metadata) %>

‎apps/transport/lib/transport_web/templates/validation/show_netex_v0_1_0.html.heex‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,16 @@
4040
<div class="validation-content-details">
4141
<div class="panel">
4242
<%= if has_errors?(@validation_summary) do %>
43-
<%= pagination_links(@conn, @issues, [@validation_id],
44-
issue_type: @results_adapter.issue_type(@issues.entries),
45-
token: @token,
46-
path: &validation_path/4,
47-
action: :show
48-
) %>
49-
<%= render(netex_template(), issues: @issues || [], conn: @conn) %>
50-
<div class="pt-24">
51-
<%= pagination_links(@conn, @issues, [@validation_id],
52-
issue_type: @results_adapter.issue_type(@issues.entries),
43+
<% pagination =
44+
pagination_links(@conn, @issues, [@validation_id],
45+
issue_type: @filter["issue_type"],
5346
token: @token,
5447
path: &validation_path/4,
5548
action: :show
5649
) %>
57-
</div>
50+
<%= pagination %>
51+
<%= render(netex_template(), issues: @issues || [], conn: @conn) %>
52+
<div class="pt-24"><%= pagination %></div>
5853
<% else %>
5954
<h2><%= dgettext("validations", "Nice work, there are no issues!") %></h2>
6055
<% end %>

‎apps/transport/lib/transport_web/templates/validation/show_netex_v0_2_x.html.heex‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<div class="validation-content">
2323
<div id="issues" class="container">
2424
<% %{"worst_occurrences" => worst_occurrences} = @max_severity %>
25-
<% current_category = @conn.params["issues_category"] || "" %>
25+
<% current_category = @filter["issues_category"] || "" %>
2626
<div id="issues" class="validation-navigation">
2727
<nav class="issues-list validation" role="navigation">
2828
<.netex_validation_summary

‎apps/transport/lib/validators/netex/results_adapter.ex‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ defmodule Transport.Validators.NeTEx.ResultsAdapter do
1212
@callback count_max_severity(map()) :: {binary(), integer()}
1313
@callback no_error?(binary()) :: boolean()
1414
@callback french_profile_compliance_check() :: :none | :partial | :good_enough
15+
@callback to_dataframe(list()) :: Explorer.DataFrame.t()
16+
@callback to_binary_result(list()) :: binary()
1517

1618
def resolve("0.2.1"), do: Transport.Validators.NeTEx.ResultsAdapters.V0_2_1
1719
def resolve("0.2.0"), do: Transport.Validators.NeTEx.ResultsAdapters.V0_2_0

0 commit comments

Comments
 (0)