Skip to content

Commit b77d480

Browse files
vieiralucasvieiralucas
committed
only iterate datapoints once, limit group by to 100 groups
1 parent aad57b0 commit b77d480

File tree

3 files changed

+90
-77
lines changed

3 files changed

+90
-77
lines changed

‎apps/api/src/python/visualizations-v2.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,13 @@ def _briefer_create_visualization(df, options):
205205
capped = True
206206
result = result.head(50000)
207207
208+
if series["groupBy"]:
209+
groups = result[series["groupBy"]["name"]].unique()
210+
if len(groups) > 100:
211+
capped = True
212+
groups = groups[:100]
213+
result = result[result[series["groupBy"]["name"]].isin(groups)]
214+
208215
return result, capped
209216
210217
def apply_filters(df, filters):
@@ -412,6 +419,18 @@ def _briefer_create_visualization(df, options):
412419
413420
y_name = series["id"]
414421
422+
# Group rows by their group value first to avoid iterating through all rows for each group
423+
grouped_rows = {}
424+
if series["groupBy"]:
425+
for _, row in series_dataframe.iterrows():
426+
group_value = row[series["groupBy"]["name"]]
427+
if group_value not in grouped_rows:
428+
grouped_rows[group_value] = []
429+
grouped_rows[group_value].append(row)
430+
else:
431+
# Store just the rows without the index for consistency
432+
grouped_rows[None] = [row for _, row in series_dataframe.iterrows()]
433+
415434
for group in groups:
416435
color_index += 1
417436
dataset_index = len(data["dataset"])
@@ -426,10 +445,9 @@ def _briefer_create_visualization(df, options):
426445
"source": [],
427446
}
428447
429-
for _, row in series_dataframe.iterrows():
430-
if group is not None and row[series["groupBy"]["name"]] != group:
431-
continue
432-
448+
# Process only rows for this specific group
449+
group_rows = grouped_rows.get(group, [])
450+
for row in group_rows: # No need to unpack since we're just storing rows
433451
y_value = row[y_name]
434452
row_data = {}
435453
@@ -446,7 +464,6 @@ def _briefer_create_visualization(df, options):
446464
y_value = y_value / total if total != 0 else 1
447465
448466
row_data[y_name] = y_value
449-
450467
dataset["source"].append(row_data)
451468
452469
data["dataset"].append(dataset)

‎apps/web/src/components/v2Editor/customBlocks/visualizationV2/VisualizationView.tsx‎

Lines changed: 65 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import {
3535
VisualizationV2BlockOutputResult,
3636
XAxis,
3737
} from '@briefer/editor'
38-
import { head } from 'ramda'
38+
import { head, uniq } from 'ramda'
3939

4040
const FONT_FAMILY = ['Inter', ...twFontFamiliy.sans].join(', ')
4141

@@ -90,7 +90,7 @@ function VisualizationViewV2(props: Props) {
9090
<LargeSpinner color="#b8f229" />
9191
</div>
9292
)}
93-
{!props.tooManyDataPointsHidden && !props.hasControls && (
93+
{!props.tooManyDataPointsHidden && props.hasControls && (
9494
<div className="absolute top-0 left-0 right-0 bg-yellow-50 p-2">
9595
<div className="flex items-center justify-center gap-x-2">
9696
<ExclamationTriangleIcon className="h-4 w-4 text-yellow-500" />
@@ -415,68 +415,68 @@ function BrieferResult(props: {
415415
}
416416
})()
417417

418-
return `
419-
<div>
420-
${xFormatted}
421-
<div>
422-
${params
423-
.map((param, i) => {
424-
const dataset = props.result.dataset[i]
425-
const row = dataset.source[param.dataIndex]
426-
let result = ''
427-
for (const [key, value] of Object.entries(row)) {
428-
if (key === props.input.xAxis?.name) {
429-
continue
430-
}
431-
432-
let seriesInput: SeriesV2 | null = null
433-
for (const yAxis of props.input.yAxes) {
434-
for (const series of yAxis.series) {
435-
if (series.id === key) {
436-
seriesInput = series
437-
break
438-
}
439-
}
440-
}
441-
442-
let formattedValue = value
443-
if (seriesInput?.column) {
444-
if (
445-
NumpyDateTypes.safeParse(seriesInput.column.type)
446-
.success
447-
) {
448-
formattedValue = formatDateTime(
449-
value,
450-
seriesInput.dateFormat
451-
)
452-
} else if (
453-
NumpyNumberTypes.safeParse(seriesInput.column.type)
454-
.success &&
455-
typeof value === 'number' &&
456-
seriesInput.numberFormat
457-
) {
458-
formattedValue = formatNumber(
459-
value,
460-
seriesInput.numberFormat
461-
)
462-
}
463-
}
418+
let yValues = ''
419+
let counter = 0
420+
for (const [i, param] of Array.from(params.entries())) {
421+
if (counter > 15) {
422+
break
423+
}
424+
425+
const dataset = props.result.dataset[i]
426+
const row = dataset.source[param.dataIndex] ?? []
427+
let result = ''
428+
for (const [key, value] of Object.entries(row)) {
429+
if (
430+
key === props.input.xAxis?.name ||
431+
value === 0 ||
432+
value === ''
433+
) {
434+
continue
435+
}
436+
437+
let seriesInput: SeriesV2 | null = null
438+
for (const yAxis of props.input.yAxes) {
439+
for (const series of yAxis.series) {
440+
if (series.id === key) {
441+
seriesInput = series
442+
break
443+
}
444+
}
445+
}
446+
447+
let formattedValue = value
448+
if (seriesInput?.column) {
449+
if (NumpyDateTypes.safeParse(seriesInput.column.type).success) {
450+
formattedValue = formatDateTime(value, seriesInput.dateFormat)
451+
} else if (
452+
NumpyNumberTypes.safeParse(seriesInput.column.type).success &&
453+
typeof value === 'number' &&
454+
seriesInput.numberFormat
455+
) {
456+
formattedValue = formatNumber(value, seriesInput.numberFormat)
457+
}
458+
}
464459

465-
result += `<div style="display: flex; align-items: center; justify-content: space-between; gap: 20px;">
460+
result += `<div style="display: flex; align-items: center; justify-content: space-between; gap: 20px;">
466461
<div>${param.marker ?? ''}${param.seriesName ?? key}</div>
467462
<div>${formattedValue}</div>
468463
</div>`
469-
}
464+
counter++
465+
}
466+
467+
yValues += result
468+
}
470469

471-
return result
472-
})
473-
.join('')}
470+
return `
471+
<div>
472+
${xFormatted}
473+
<div>${yValues}</div>
474474
</div>
475475
`
476476
},
477477
},
478478
}
479-
}, [props.result, props.input, size])
479+
}, [props.result, props.input, props.hasControls, props.title])
480480

481481
if (!size) {
482482
return <div className="w-full h-full" ref={measureDiv} />
@@ -546,8 +546,8 @@ function Echarts(props: EchartsProps) {
546546
const xAxes = Array.isArray(props.option.xAxis)
547547
? props.option.xAxis
548548
: props.option.xAxis
549-
? [props.option.xAxis]
550-
: []
549+
? [props.option.xAxis]
550+
: []
551551
let isRotated = false
552552
const nextXAxes = xAxes.map((xAxis) => {
553553
if (!xAxis || xAxis.type !== 'category') {
@@ -599,10 +599,10 @@ function Echarts(props: EchartsProps) {
599599
left: isRotated ? '60' : props.option.grid.left,
600600
}
601601
: isRotated
602-
? {
603-
left: '60',
604-
}
605-
: undefined,
602+
? {
603+
left: '60',
604+
}
605+
: undefined,
606606
})
607607
setIsReady(true)
608608
hiddenChart.dispose()
@@ -911,9 +911,11 @@ function getValueAxis(
911911

912912
let min = -Infinity
913913
let max = Infinity
914-
const xFields = result.series
915-
.map((s) => s.encode?.x)
916-
.filter((x): x is string | number => x !== undefined)
914+
const xFields = uniq(
915+
result.series
916+
.map((s) => s.encode?.x)
917+
.filter((x): x is string | number => x !== undefined)
918+
)
917919
const values = result.dataset
918920
.flatMap((d) =>
919921
xFields.flatMap((f) => d.source.flatMap((r) => r[f.toString()]))

‎apps/web/src/components/v2Editor/customBlocks/visualizationV2/index.tsx‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ function VisualizationBlockV2(props: Props) {
468468
[props.block]
469469
)
470470

471-
const tooManyDataPointsHidden = !(attrs.output?.tooManyDataPoints ?? false)
471+
const tooManyDataPointsHidden = !(attrs.output?.tooManyDataPoints ?? true)
472472

473473
const onHideTooManyDataPointsWarning = useCallback(() => {
474474
if (!attrs.output) {
@@ -528,10 +528,7 @@ function VisualizationBlockV2(props: Props) {
528528
// Check if only formatting fields changed by comparing old and new values
529529
// excluding both filters and formatting fields
530530
const oldValueForComparison = {
531-
...omit(
532-
[...xAxisFormattingFields, 'filters'],
533-
val.oldValue
534-
),
531+
...omit([...xAxisFormattingFields, 'filters'], val.oldValue),
535532
yAxes: val.oldValue.yAxes.map((yAxis: YAxisV2) => ({
536533
...yAxis,
537534
series: yAxis.series.map((series) => ({
@@ -540,10 +537,7 @@ function VisualizationBlockV2(props: Props) {
540537
})),
541538
}
542539
const newValueForComparison = {
543-
...omit(
544-
[...xAxisFormattingFields, 'filters'],
545-
input
546-
),
540+
...omit([...xAxisFormattingFields, 'filters'], input),
547541
yAxes: input.yAxes.map((yAxis: YAxisV2) => ({
548542
...yAxis,
549543
series: yAxis.series.map((series: SeriesV2) => ({

0 commit comments

Comments
 (0)