Skip to content

Add timeZone to ScheduledSparkApplication #2471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 31, 2025
Merged

Add timeZone to ScheduledSparkApplication #2471

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
230263d
Add timeZone to ScheduledSparkApplication
jacobsalway Mar 16, 2025
725f262
Update api/v1beta2/scheduledsparkapplication_types.go
jacobsalway Mar 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions api/v1beta2/scheduledsparkapplication_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ type ScheduledSparkApplicationSpec struct {

// Schedule is a cron schedule on which the application should run.
Schedule string `json:"schedule"`
// TimeZone is the time zone in which the cron schedule will be interpreted in.
// This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
// or a valid IANA location name e.g. "America/New_York".
// +optional
// Defaults to "Local".
TimeZone string `json:"timeZone"`
// Template is a template from which SparkApplication instances can be created.
Template SparkApplicationSpec `json:"template"`
// Suspend is a flag telling the controller to suspend subsequent runs of the application if set to true.
Expand Down Expand Up @@ -80,6 +86,7 @@ type ScheduledSparkApplicationStatus struct {
// +kubebuilder:resource:scope=Namespaced,shortName=scheduledsparkapp,singular=scheduledsparkapplication
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:JSONPath=.spec.schedule,name=Schedule,type=string
// +kubebuilder:printcolumn:JSONPath=.spec.timeZone,name=TimeZone,type=string
// +kubebuilder:printcolumn:JSONPath=.spec.suspend,name=Suspend,type=string
// +kubebuilder:printcolumn:JSONPath=.status.lastRun,name=Last Run,type=date
// +kubebuilder:printcolumn:JSONPath=.status.lastRunName,name=Last Run Name,type=string
Expand Down
10 changes: 10 additions & 0 deletions charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ spec:
- jsonPath: .spec.schedule
name: Schedule
type: string
- jsonPath: .spec.timeZone
name: TimeZone
type: string
- jsonPath: .spec.suspend
name: Suspend
type: string
Expand Down Expand Up @@ -12384,6 +12387,13 @@ spec:
- sparkVersion
- type
type: object
timeZone:
description: |-
TimeZone is the time zone in which the cron schedule will be interpreted in.
This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
or a valid IANA location name e.g. "America/New_York".
Defaults to "Local".
type: string
required:
- schedule
- template
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ spec:
- jsonPath: .spec.schedule
name: Schedule
type: string
- jsonPath: .spec.timeZone
name: TimeZone
type: string
- jsonPath: .spec.suspend
name: Suspend
type: string
Expand Down Expand Up @@ -12384,6 +12387,13 @@ spec:
- sparkVersion
- type
type: object
timeZone:
description: |-
TimeZone is the time zone in which the cron schedule will be interpreted in.
This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
or a valid IANA location name e.g. "America/New_York".
Defaults to "Local".
type: string
required:
- schedule
- template
Expand Down
30 changes: 30 additions & 0 deletions docs/api-docs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1412,6 +1412,21 @@ string
</tr>
<tr>
<td>
<code>timeZone</code><br/>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>TimeZone is the time zone in which the cron schedule will be interpreted in.
This value is passed to time.LoadLocation, so it must be either &ldquo;Local&rdquo;, &ldquo;UTC&rdquo;,
or a valid IANA location name e.g. &ldquo;America/New_York&rdquo;.
Defaults to &ldquo;Local&rdquo;.</p>
</td>
</tr>
<tr>
<td>
<code>template</code><br/>
<em>
<a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationSpec">
Expand Down Expand Up @@ -1521,6 +1536,21 @@ string
</tr>
<tr>
<td>
<code>timeZone</code><br/>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>TimeZone is the time zone in which the cron schedule will be interpreted in.
This value is passed to time.LoadLocation, so it must be either &ldquo;Local&rdquo;, &ldquo;UTC&rdquo;,
or a valid IANA location name e.g. &ldquo;America/New_York&rdquo;.
Defaults to &ldquo;Local&rdquo;.</p>
</td>
</tr>
<tr>
<td>
<code>template</code><br/>
<em>
<a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationSpec">
Expand Down
28 changes: 27 additions & 1 deletion internal/controller/scheduledsparkapplication/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ import (
"fmt"
"reflect"
"sort"
"strings"
"time"

_ "time/tzdata"

"github.com/robfig/cron/v3"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -107,7 +110,30 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, nil
}

schedule, parseErr := cron.ParseStandard(scheduledApp.Spec.Schedule)
timezone := scheduledApp.Spec.TimeZone
if timezone == "" {
timezone = "Local"
} else {
// Explicitly validate the timezone for a better user experience, but only if it's explicitly specified
_, err = time.LoadLocation(timezone)
if err != nil {
logger.Error(err, "Failed to load timezone location", "name", scheduledApp.Name, "namespace", scheduledApp.Namespace, "timezone", timezone)
scheduledApp.Status.ScheduleState = v1beta2.ScheduleStateFailedValidation
scheduledApp.Status.Reason = fmt.Sprintf("Invalid timezone: %v", err)
if updateErr := r.updateScheduledSparkApplicationStatus(ctx, scheduledApp); updateErr != nil {
return ctrl.Result{Requeue: true}, updateErr
}
return ctrl.Result{}, nil
}
}

// Ensure backwards compatibility if the schedule is relying on internal functionality of robfig/cron
cronSchedule := scheduledApp.Spec.Schedule
if !strings.HasPrefix(cronSchedule, "CRON_TZ=") && !strings.HasPrefix(cronSchedule, "TZ=") {
cronSchedule = fmt.Sprintf("CRON_TZ=%s %s", timezone, cronSchedule)
}

schedule, parseErr := cron.ParseStandard(cronSchedule)
if parseErr != nil {
logger.Error(err, "Failed to parse schedule of ScheduledSparkApplication", "name", scheduledApp.Name, "namespace", scheduledApp.Namespace, "schedule", scheduledApp.Spec.Schedule)
scheduledApp.Status.ScheduleState = v1beta2.ScheduleStateFailedValidation
Expand Down