ok  	github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation	0.031s	coverage: 13.7% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2	[no test files]
=== RUN   TestGenGeneralName
--- PASS: TestGenGeneralName (0.00s)
PASS
coverage: 0.5% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/common/jobcontroller	0.021s	coverage: 0.5% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured	[no test files]
=== RUN   TestCreatePods
--- PASS: TestCreatePods (0.01s)
=== RUN   TestCreateService
time="2019-03-18T01:41:51Z" level=info msg="Controller test-tfjob created service empty_service"
--- PASS: TestCreateService (0.00s)
=== RUN   TestCreateServicesWithControllerRef
time="2019-03-18T01:41:51Z" level=info msg="Controller test-tfjob created service empty_service"
--- PASS: TestCreateServicesWithControllerRef (0.00s)
=== RUN   TestClaimServices
--- PASS: TestClaimServices (0.00s)
PASS
coverage: 44.9% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/control	0.055s	coverage: 44.9% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
=== RUN   TestNormalPath
time="2019-03-18T01:41:58Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:41:58Z" level=info msg="Creating Job controller"
time="2019-03-18T01:41:58Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (2.494876ms)" job=default.test-tfjob
time="2019-03-18T01:41:58Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:41:58Z" level=info msg="Creating Job controller"
time="2019-03-18T01:41:58Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (470.056µs)" job=default.test-tfjob
time="2019-03-18T01:41:58Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:41:58Z" level=info msg="Creating Job controller"
time="2019-03-18T01:41:58Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (388.294µs)" job=default.test-tfjob
time="2019-03-18T01:41:58Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:41:58Z" level=info msg="Creating Job controller"
time="2019-03-18T01:41:58Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:41:58Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:41:58Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (468.416µs)" job=default.test-tfjob
time="2019-03-18T01:41:58Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:41:58Z" level=info msg="Creating Job controller"
time="2019-03-18T01:41:58Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:41:58Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
E0318 01:41:58.601112    9691 event.go:259] Could not construct reference to: '&v1beta1.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta1.TFJobSpec{CleanPodPolicy:(*v1beta1.CleanPodPolicy)(0xc4204cc270), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta1.TFReplicaType]*v1beta1.ReplicaSpec{"PS":(*v1beta1.ReplicaSpec)(0xc42044fb80), "Worker":(*v1beta1.ReplicaSpec)(0xc420624000)}}, Status:v1beta1.JobStatus{Conditions:[]v1beta1.JobCondition(nil), ReplicaStatuses:map[v1beta1.ReplicaType]*v1beta1.ReplicaStatus{"PS":(*v1beta1.ReplicaStatus)(0xc420379560), "Worker":(*v1beta1.ReplicaStatus)(0xc420379590)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'.   Will not report event: 'Normal' 'ExitedWithCode' 'Pod: default.worker-0 exited with code 130'
time="2019-03-18T01:42:01Z" level=info msg="Need to restart the pod: default.worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:01.329691    9691 event.go:259] Could not construct reference to: '&v1beta1.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta1.TFJobSpec{CleanPodPolicy:(*v1beta1.CleanPodPolicy)(0xc420324ff0), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta1.TFReplicaType]*v1beta1.ReplicaSpec{"Worker":(*v1beta1.ReplicaSpec)(0xc4208ed600)}}, Status:v1beta1.JobStatus{Conditions:[]v1beta1.JobCondition(nil), ReplicaStatuses:map[v1beta1.ReplicaType]*v1beta1.ReplicaStatus{"Worker":(*v1beta1.ReplicaStatus)(0xc42080fce0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Warning' 'TFJobRestarting' 'TFJob test-tfjob is restarting because 1 Worker replica(s) failed.'
time="2019-03-18T01:42:01Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:01Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (427.328µs)" job=default.test-tfjob
--- PASS: TestExitCode (0.00s)
=== RUN   TestAddService
time="2019-03-18T01:42:01Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:01Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:01Z" level=info msg="Started workers"
time="2019-03-18T01:42:01Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:01Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:01Z" level=info msg="Started workers"
--- PASS: TestAddService (0.10s)
=== RUN   TestFailed
time="2019-03-18T01:42:01Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:01.432853    9691 event.go:259] Could not construct reference to: '&v1beta1.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta1.TFJobSpec{CleanPodPolicy:(*v1beta1.CleanPodPolicy)(nil), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta1.TFReplicaType]*v1beta1.ReplicaSpec{"Worker":(*v1beta1.ReplicaSpec)(0xc4207ec2c0)}}, Status:v1beta1.JobStatus{Conditions:[]v1beta1.JobCondition(nil), ReplicaStatuses:map[v1beta1.ReplicaType]*v1beta1.ReplicaStatus{"Worker":(*v1beta1.ReplicaStatus)(0xc42046ea00)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'TFJobFailed' 'TFJob test-tfjob has failed because 1 Worker replica(s) failed.'
--- PASS: TestFailed (0.00s)
=== RUN   TestStatus
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=2, failed=2" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=2, running=0, failed=2" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=3, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=1, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:01Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:01Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
--- PASS: TestStatus (0.02s)
=== RUN   TestGenOwnerReference
--- PASS: TestGenOwnerReference (0.00s)
=== RUN   TestGenLabels
--- PASS: TestGenLabels (0.00s)
=== RUN   TestConvertTFJobToUnstructured
--- PASS: TestConvertTFJobToUnstructured (0.00s)
PASS
coverage: 42.7% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow	2.913s	coverage: 42.7% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
=== RUN   TestNormalPath
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (1.979755ms)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (529.889µs)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (425.973µs)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (489.951µs)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
E0318 01:42:07.640129    9800 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc42005a1e0), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc42046edc0), "PS":(*v1beta2.ReplicaSpec)(0xc42046eb00)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"PS":(*v1beta2.ReplicaStatus)(0xc4201cde20), "Worker":(*v1beta2.ReplicaStatus)(0xc4201cde50)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'TFJobSucceeded' 'TFJob test-tfjob successfully completed.'
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (2.396319ms)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (294.739µs)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: ps-0" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: ps-0" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-1" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-1" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (710.609µs)" job=default.test-tfjob
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:07Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (368.524µs)" job=default.test-tfjob
--- PASS: TestNormalPath (0.02s)
=== RUN   TestRun
time="2019-03-18T01:42:07Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:07Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:07Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:07Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:07Z" level=info msg="Started workers"
time="2019-03-18T01:42:08Z" level=info msg="Shutting down workers"
--- PASS: TestRun (0.50s)
=== RUN   TestSyncPdb
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
--- PASS: TestSyncPdb (0.00s)
=== RUN   TestAddTFJob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:08Z" level=info msg="TFJob test-tfjob is created." job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:08Z" level=info msg="Started workers"
--- PASS: TestAddTFJob (0.10s)
=== RUN   TestCopyLabelsAndAnnotation
time="2019-03-18T01:42:08Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:08Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (360.284µs)" job=default.test-tfjob
--- PASS: TestCopyLabelsAndAnnotation (0.00s)
=== RUN   TestDeletePodsAndServices
time="2019-03-18T01:42:08Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:08Z" level=info msg="Started workers"
time="2019-03-18T01:42:08Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (280.178µs)" job=default.test-tfjob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (323.468µs)" job=default.test-tfjob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (850.827µs)" job=default.test-tfjob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (274.287µs)" job=default.test-tfjob
--- PASS: TestDeletePodsAndServices (0.01s)
=== RUN   TestCleanupTFJob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (349.311µs)" job=default.test-tfjob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:08Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:08Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (384.268µs)" job=default.test-tfjob
time="2019-03-18T01:42:08Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:08Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:10Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T01:42:10Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (645.327µs)" job=default.test-tfjob
--- PASS: TestCleanupTFJob (2.01s)
=== RUN   TestActiveDeadlineSeconds
time="2019-03-18T01:42:10Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:10Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:10Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (643.577µs)" job=default.test-tfjob
time="2019-03-18T01:42:10Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (288.17µs)" job=default.test-tfjob
time="2019-03-18T01:42:10Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:10Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:10Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="Job with ActiveDeadlineSeconds will sync after 2 seconds" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:10Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (500.904µs)" job=default.test-tfjob
time="2019-03-18T01:42:12Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Job with ActiveDeadlineSeconds will sync after 2 seconds" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (714.492µs)" job=default.test-tfjob
--- FAIL: TestActiveDeadlineSeconds (2.00s)
	job_test.go:645: 4 workers and 2 ps is running, ActiveDeadlineSeconds is 10: unexpected status. Expected true, saw false
=== RUN   TestAddPod
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:12Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:12Z" level=info msg="Started workers"
--- PASS: TestAddPod (0.10s)
=== RUN   TestClusterSpec
time="2019-03-18T01:42:12Z" level=info msg="Shutting down workers"
--- PASS: TestClusterSpec (0.00s)
=== RUN   TestRestartPolicy
--- PASS: TestRestartPolicy (0.00s)
=== RUN   TestExitCode
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Ignoring inactive pod default/worker-0 in state Failed, deletion time <nil>"
time="2019-03-18T01:42:12Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:12Z" level=info msg="Pod: default.worker-0 exited with code 130" job=default.test-tfjob replica-type=worker uid=
E0318 01:42:12.373558    9800 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc42020e730), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc42038f8c0)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc42002b330)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'ExitedWithCode' 'Pod: default.worker-0 exited with code 130'
time="2019-03-18T01:42:12Z" level=info msg="Need to restart the pod: default.worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:12.373699    9800 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc42020e730), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc42038f8c0)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc42002b330)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Warning' 'TFJobRestarting' 'TFJob test-tfjob is restarting because 1 Worker replica(s) failed.'
time="2019-03-18T01:42:12Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:12Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (740.164µs)" job=default.test-tfjob
--- PASS: TestExitCode (0.00s)
=== RUN   TestAddService
time="2019-03-18T01:42:12Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:12Z" level=info msg="Started workers"
time="2019-03-18T01:42:12Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:12Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:12Z" level=info msg="Started workers"
--- PASS: TestAddService (0.10s)
=== RUN   TestFailed
time="2019-03-18T01:42:12Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:12.476326    9800 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(nil), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc420242580)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc42037bae0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'TFJobFailed' 'TFJob test-tfjob has failed because 1 Worker replica(s) failed.'
--- PASS: TestFailed (0.00s)
=== RUN   TestStatus
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=2, failed=2" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=2, running=0, failed=2" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=3, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=1, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:12Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=4" job=default.test-tfjob uid=
time="2019-03-18T01:42:12Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
--- PASS: TestStatus (0.01s)
=== RUN   TestGenOwnerReference
--- PASS: TestGenOwnerReference (0.00s)
=== RUN   TestGenLabels
--- PASS: TestGenLabels (0.00s)
=== RUN   TestConvertTFJobToUnstructured
--- PASS: TestConvertTFJobToUnstructured (0.00s)
FAIL
coverage: 42.8% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
FAIL	github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow	4.910s
exit status 1: warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/util
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/util/signals
warning: no packages being tested depend on matches for pattern github.com/kubeflow/tf-operator/pkg/version

travis_time:end:14b44738:start=1552873275285709843,finish=1552873332506466627,duration=57220756784

TravisBuddy Request Identifier: 40e50f60-4922-11e9-ab4c-bfa38af62fb3

Travis tests have failed
Hey @ChanYiLin,
Please read the following log in order to understand the failure reason.
It'll be awesome if you fix what's wrong and commit the changes.
1st Build
View build log

     goveralls -service=travis-ci -v -package ./pkg/... -ignore "pkg/client/*/*.go,pkg/client/*/*/*.go,pkg/client/*/*/*/*.go,pkg/client/*/*/*/*/*.go,pkg/client/*/*/*/*/*/*.go,pkg/client/*/*/*/*/*/*/*.go,pkg/util/testutil/*.go,pkg/apis/tensorflow/*/zz_generated.*.go,pkg/apis/tensorflow/*/*_generated.go,pkg/apis/common/*/zz_generated.*.go,pkg/apis/common/*/*_generated.go"

?   	github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2	[no test files]
=== RUN   TestSetTypeNames
--- PASS: TestSetTypeNames (0.00s)
=== RUN   TestSetDefaultTFJob
--- PASS: TestSetDefaultTFJob (0.00s)
=== RUN   TestIsChieforMaster
--- PASS: TestIsChieforMaster (0.00s)
PASS
coverage: 2.3% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1	0.055s
=== RUN   TestSetTypeNames
--- PASS: TestSetTypeNames (0.00s)
=== RUN   TestSetDefaultTFJob
--- PASS: TestSetDefaultTFJob (0.00s)
=== RUN   TestIsChieforMaster
--- PASS: TestIsChieforMaster (0.00s)
PASS
coverage: 2.3% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2	0.052s
=== RUN   TestValidateBetaOneTFJobSpec
--- PASS: TestValidateBetaOneTFJobSpec (0.00s)
=== RUN   TestValidateBetaTwoTFJobSpec
--- PASS: TestValidateBetaTwoTFJobSpec (0.00s)
PASS
coverage: 2.9% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation	0.045s
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2	[no test files]
=== RUN   TestGenGeneralName
--- PASS: TestGenGeneralName (0.00s)
PASS
coverage: 1.0% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/common/jobcontroller	0.047s
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil	[no test files]
?   	github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured	[no test files]
=== RUN   TestCreatePods
--- PASS: TestCreatePods (0.01s)
=== RUN   TestCreateService
--- PASS: TestCreateService (0.00s)
=== RUN   TestCreateServicesWithControllerRef
--- PASS: TestCreateServicesWithControllerRef (0.00s)
=== RUN   TestClaimServices
--- PASS: TestClaimServices (0.00s)
PASS
coverage: 4.4% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/control	0.063s
=== RUN   TestNormalPath
--- PASS: TestNormalPath (0.03s)
=== RUN   TestRun
--- PASS: TestRun (0.50s)
=== RUN   TestSyncPdb
--- PASS: TestSyncPdb (0.00s)
=== RUN   TestAddTFJob
--- PASS: TestAddTFJob (0.10s)
=== RUN   TestCopyLabelsAndAnnotation
--- PASS: TestCopyLabelsAndAnnotation (0.00s)
=== RUN   TestDeletePodsAndServices
--- PASS: TestDeletePodsAndServices (0.00s)
=== RUN   TestCleanupTFJob
--- PASS: TestCleanupTFJob (2.00s)
=== RUN   TestAddPod
--- PASS: TestAddPod (0.10s)
=== RUN   TestClusterSpec
--- PASS: TestClusterSpec (0.00s)
=== RUN   TestRestartPolicy
--- PASS: TestRestartPolicy (0.00s)
=== RUN   TestExitCode
--- PASS: TestExitCode (0.00s)
=== RUN   TestAddService
--- PASS: TestAddService (0.10s)
=== RUN   TestFailed
--- PASS: TestFailed (0.00s)
=== RUN   TestStatus
--- PASS: TestStatus (0.01s)
=== RUN   TestGenOwnerReference
--- PASS: TestGenOwnerReference (0.00s)
=== RUN   TestGenLabels
--- PASS: TestGenLabels (0.00s)
=== RUN   TestConvertTFJobToUnstructured
--- PASS: TestConvertTFJobToUnstructured (0.00s)
PASS
coverage: 30.0% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
ok  	github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow	2.907s
=== RUN   TestNormalPath
--- PASS: TestNormalPath (0.02s)
=== RUN   TestRun
--- PASS: TestRun (0.50s)
=== RUN   TestSyncPdb
--- PASS: TestSyncPdb (0.00s)
=== RUN   TestAddTFJob
--- PASS: TestAddTFJob (0.10s)
=== RUN   TestCopyLabelsAndAnnotation
--- PASS: TestCopyLabelsAndAnnotation (0.00s)
=== RUN   TestDeletePodsAndServices
--- PASS: TestDeletePodsAndServices (0.01s)
=== RUN   TestCleanupTFJob
--- PASS: TestCleanupTFJob (2.00s)
=== RUN   TestActiveDeadlineSeconds
--- FAIL: TestActiveDeadlineSeconds (2.00s)
	job_test.go:645: 4 workers and 2 ps is running, ActiveDeadlineSeconds is 10: unexpected status. Expected true, saw false
=== RUN   TestAddPod
--- PASS: TestAddPod (0.10s)
=== RUN   TestClusterSpec
--- PASS: TestClusterSpec (0.00s)
=== RUN   TestRestartPolicy
--- PASS: TestRestartPolicy (0.00s)
=== RUN   TestExitCode
--- PASS: TestExitCode (0.00s)
=== RUN   TestAddService
--- PASS: TestAddService (0.10s)
=== RUN   TestFailed
--- PASS: TestFailed (0.00s)
=== RUN   TestStatus
--- PASS: TestStatus (0.01s)
=== RUN   TestGenOwnerReference
--- PASS: TestGenOwnerReference (0.00s)
=== RUN   TestGenLabels
--- PASS: TestGenLabels (0.00s)
=== RUN   TestConvertTFJobToUnstructured
--- PASS: TestConvertTFJobToUnstructured (0.00s)
FAIL
coverage: 31.9% of statements in github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/common/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/apis/tensorflow/validation, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/scheme, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/internalinterfaces, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/informers/externalversions/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta1, github.com/kubeflow/tf-operator/pkg/client/listers/tensorflow/v1beta2, github.com/kubeflow/tf-operator/pkg/common/jobcontroller, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil, github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/unstructured, github.com/kubeflow/tf-operator/pkg/control, github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow, github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow, github.com/kubeflow/tf-operator/pkg/logger, github.com/kubeflow/tf-operator/pkg/util, github.com/kubeflow/tf-operator/pkg/util/k8sutil, github.com/kubeflow/tf-operator/pkg/util/signals, github.com/kubeflow/tf-operator/pkg/util/train, github.com/kubeflow/tf-operator/pkg/version
exit status 1
FAIL	github.com/kubeflow/tf-operator/pkg/controller.v1beta2/tensorflow	4.907s
exit status 1: warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/fake
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta1/fake
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/client/clientset/versioned/typed/tensorflow/v1beta2/fake
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/testutil
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/common/util/v1beta1/unstructured
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/common/util/v1beta2/testutil
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/controller.v1beta1/tensorflow
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/util
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/util/signals
warning: no packages being tested depend on github.com/kubeflow/tf-operator/pkg/version
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (1.702949ms)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (338.534µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (492.001µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (408.207µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: ps-0" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: ps-0" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-1" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-1" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-2" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (1.421026ms)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=2, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=4, running=4, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (347.141µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=2, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: ps-1" job=default.test-tfjob replica-type=ps uid=
time="2019-03-18T02:06:45Z" level=info msg="Need to create new pod: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="need to create new service: worker-3" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (479.429µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
E0318 02:06:45.849207    5873 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc4203c6f30), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"PS":(*v1beta2.ReplicaSpec)(0xc4202e3b80), "Worker":(*v1beta2.ReplicaSpec)(0xc4203d4000)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc4204f02e0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'TFJobSucceeded' 'TFJob test-tfjob successfully completed.'
time="2019-03-18T02:06:45Z" level=info msg="TFJob=test-tfjob, ReplicaType=PS expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:45Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (808.439µs)" job=default.test-tfjob
time="2019-03-18T02:06:45Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:45Z" level=info msg="Starting TFJob controller"
time="2019-03-18T02:06:45Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T02:06:45Z" level=info msg="Starting 1 workers"
time="2019-03-18T02:06:45Z" level=info msg="Started workers"
time="2019-03-18T02:06:46Z" level=info msg="Shutting down workers"
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Starting TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T02:06:46Z" level=info msg="TFJob test-tfjob is created." job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Starting 1 workers"
time="2019-03-18T02:06:46Z" level=info msg="Started workers"
time="2019-03-18T02:06:46Z" level=info msg="Shutting down workers"
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Starting TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Need to create new pod: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:46Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (319.786µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Starting 1 workers"
time="2019-03-18T02:06:46Z" level=info msg="Started workers"
time="2019-03-18T02:06:46Z" level=info msg="Shutting down workers"
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (280.495µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (246.811µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (692.277µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/ps-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/ps-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-0 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-1 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-2 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Ignoring inactive pod default/worker-3 in state Succeeded, deletion time <nil>"
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (653.732µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (193.403µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:46Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:46Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (180.824µs)" job=default.test-tfjob
time="2019-03-18T02:06:46Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:46Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:48Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:48Z" leime="2019-03-18T02:06:50Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (574.641µs)" job=default.test-tfjob
time="2019-03-18T02:06:50Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:50Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:50Z" level=info msg="Starting TFJob controller"
time="2019-03-18T02:06:50Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T02:06:50Z" level=info msg="Starting 1 workers"
time="2019-03-18T02:06:50Z" level=info msg="Started workers"
time="2019-03-18T02:06:50Z" level=info msg="Shutting down workers"
time="2019-03-18T02:06:50Z" level=info msg="Creating TFJob controller"
time="2019-03-18T02:06:50Z" level=info msg="Creating Job controller"
time="2019-03-18T02:06:50Z" level=info msg="Starting TFJob controller"
time="2019-03-18T02:06:50Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T02:06:50Z" level=info msg="Reconcile TFJobs test-tfjob" job=default.test-tfjob uid=
time="2019-03-18T02:06:50Z" level=info msg="Ignoring inactive pod default/worker-0 in state Failed, deletion time <nil>"
time="2019-03-18T02:06:50Z" level=info msg="Pod: default.worker-0 exited with code 130" job=default.test-tfjob replica-type=worker uid=
E0318 02:06:50.574355    5873 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc420589590), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc4208a82c0)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc4204f08c0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'ExitedWithCode' 'Pod: default.worker-0 exited with code 130'
time="2019-03-18T02:06:50Z" level=info msg="Need to restart the pod: default.worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T02:06:50Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
E0318 02:06:50.574466    5873 event.go:259] Could not construct reference to: '&v1beta2.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta2.TFJobSpec{ActiveDeadlineSeconds:(*int64)(nil), BackoffLimit:(*int32)(nil), CleanPodPolicy:(*v1beta2.CleanPodPolicy)(0xc420589590), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta2.TFReplicaType]*v1beta2.ReplicaSpec{"Worker":(*v1beta2.ReplicaSpec)(0xc4208a82c0)}}, Status:v1beta2.JobStatus{Conditions:[]v1beta2.JobCondition(nil), ReplicaStatuses:map[v1beta2.ReplicaType]*v1beta2.ReplicaStatus{"Worker":(*v1beta2.ReplicaStatus)(0xc4204f08c0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'.  Will not report event: 'Normal' 'TFJobSucceeded' 'TFJob test-tfjob successfully completed.'
time="2019-03-18T01:42:54Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (877.107µs)" job=default.test-tfjob
--- PASS: TestNormalPath (0.02s)
=== RUN   TestRun
time="2019-03-18T01:42:54Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:54Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:54Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:54Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:54Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:54Z" level=info msg="Started workers"
time="2019-03-18T01:42:55Z" level=info msg="Shutting down workers"
--- PASS: TestRun (0.50s)
=== RUN   TestSyncPdb
time="2019-03-18T01:42:55Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:55Z" level=info msg="Creating Job controller"
--- PASS: TestSyncPdb (0.00s)
=== RUN   TestAddTFJob
time="2019-03-18T01:42:55Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:55Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:55Z" level=info msg="TFJob test-tfjob is created."  Will not report event: 'Normal' 'ExitedWithCode' 'Pod: default.worker-0 exited with code 130'
time="2019-03-18T01:42:57Z" level=info msg="Need to restart the pod: default.worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:57.379126    9837 event.go:259] Could not construct reference to: '&v1beta1.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta1.TFJobSpec{CleanPodPolicy:(*v1beta1.CleanPodPolicy)(0xc42035f820), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta1.TFReplicaType]*v1beta1.ReplicaSpec{"Worker":(*v1beta1.ReplicaSpec)(0xc420300580)}}, Status:v1beta1.JobStatus{Conditions:[]v1beta1.JobCondition(nil), ReplicaStatuses:map[v1beta1.ReplicaType]*v1beta1.ReplicaStatus{"Worker":(*v1beta1.ReplicaStatus)(0xc4204b35e0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Warning' 'TFJobRestarting' 'TFJob test-tfjob is restarting because 1 Worker replica(s) failed.'
time="2019-03-18T01:42:57Z" level=info msg="need to create new service: worker-0" job=default.test-tfjob replica-type=worker uid=
time="2019-03-18T01:42:57Z" level=info msg="Finished syncing tfjob \"default/test-tfjob\" (545.392µs)" job=default.test-tfjob
time="2019-03-18T01:42:57Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:57Z" level=info msg="Started workers"
time="2019-03-18T01:42:57Z" level=info msg="Shutting down workers"
--- PASS: TestExitCode (0.00s)
=== RUN   TestAddService
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="Starting TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Waiting for informer caches to sync"
time="2019-03-18T01:42:57Z" level=info msg="Starting 1 workers"
time="2019-03-18T01:42:57Z" level=info msg="Started workers"
--- PASS: TestAddService (0.10s)
=== RUN   TestFailed
time="2019-03-18T01:42:57Z" level=info msg="Shutting down workers"
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=3, running=0, failed=1" job=default.test-tfjob uid=
E0318 01:42:57.482113    9837 event.go:259] Could not construct reference to: '&v1beta1.TFJob{TypeMeta:v1.TypeMeta{Kind:"TFJob", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"test-tfjob", GenerateName:"", Namespace:"default", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:""}, Spec:v1beta1.TFJobSpec{CleanPodPolicy:(*v1beta1.CleanPodPolicy)(nil), TTLSecondsAfterFinished:(*int32)(nil), TFReplicaSpecs:map[v1beta1.TFReplicaType]*v1beta1.ReplicaSpec{"Worker":(*v1beta1.ReplicaSpec)(0xc420301080)}}, Status:v1beta1.JobStatus{Conditions:[]v1beta1.JobCondition(nil), ReplicaStatuses:map[v1beta1.ReplicaType]*v1beta1.ReplicaStatus{"Worker":(*v1beta1.ReplicaStatus)(0xc42042fcb0)}, StartTime:(*v1.Time)(nil), CompletionTime:(*v1.Time)(nil), LastReconcileTime:(*v1.Time)(nil)}}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' 'TFJobFailed' 'TFJob test-tfjob has failed because 1 Worker replica(s) failed.'
--- PASS: TestFailed (0.00s)
=== RUN   TestStatus
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Chief expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=0, failed=1" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=0, running=0, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"
time="2019-03-18T01:42:57Z" level=info msg="Creating Job controller"
time="2019-03-18T01:42:57Z" level=info msg="TFJob=test-tfjob, ReplicaType=Worker expected=1, running=1, failed=0" job=default.test-tfjob uid=
time="2019-03-18T01:42:57Z" level=info msg="Creating TFJob controller"