2019-03-30 03:23:56 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2019-08-29 03:56:52 -04:00
|
|
|
require 'spec_helper'
|
2017-12-22 12:23:43 -05:00
|
|
|
|
2020-06-24 14:09:03 -04:00
|
|
|
RSpec.describe Clusters::Applications::Prometheus do
|
2018-09-06 06:03:38 -04:00
|
|
|
include KubernetesHelpers
|
2020-02-11 01:09:46 -05:00
|
|
|
include StubRequests
|
2018-09-06 06:03:38 -04:00
|
|
|
|
2018-03-01 18:46:02 -05:00
|
|
|
include_examples 'cluster application core specs', :clusters_applications_prometheus
|
2018-10-15 05:03:15 -04:00
|
|
|
include_examples 'cluster application status specs', :clusters_applications_prometheus
|
2019-02-07 16:40:55 -05:00
|
|
|
include_examples 'cluster application version specs', :clusters_applications_prometheus
|
2018-11-26 15:02:33 -05:00
|
|
|
include_examples 'cluster application helm specs', :clusters_applications_prometheus
|
2019-02-06 06:05:21 -05:00
|
|
|
include_examples 'cluster application initial status specs'
|
2017-12-22 12:23:43 -05:00
|
|
|
|
2019-04-15 00:02:52 -04:00
|
|
|
describe 'after_destroy' do
|
2019-12-10 02:53:40 -05:00
|
|
|
context 'cluster type is project' do
|
|
|
|
let(:cluster) { create(:cluster, :with_installed_helm) }
|
|
|
|
let(:application) { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
|
2019-04-15 00:02:52 -04:00
|
|
|
|
2019-12-10 02:53:40 -05:00
|
|
|
it 'deactivates prometheus_service after destroy' do
|
|
|
|
expect(Clusters::Applications::DeactivateServiceWorker)
|
|
|
|
.to receive(:perform_async).with(cluster.id, 'prometheus')
|
2019-04-15 00:02:52 -04:00
|
|
|
|
2019-12-10 02:53:40 -05:00
|
|
|
application.destroy!
|
|
|
|
end
|
2019-04-15 00:02:52 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-01-03 20:45:57 -05:00
|
|
|
describe 'transition to installed' do
|
|
|
|
let(:project) { create(:project) }
|
2019-12-10 02:53:40 -05:00
|
|
|
let(:cluster) { create(:cluster, :with_installed_helm) }
|
|
|
|
let(:application) { create(:clusters_applications_prometheus, :installing, cluster: cluster) }
|
2018-01-03 20:45:57 -05:00
|
|
|
|
2019-12-10 02:53:40 -05:00
|
|
|
it 'schedules post installation job' do
|
|
|
|
expect(Clusters::Applications::ActivateServiceWorker)
|
|
|
|
.to receive(:perform_async).with(cluster.id, 'prometheus')
|
2018-01-03 20:45:57 -05:00
|
|
|
|
2019-12-10 02:53:40 -05:00
|
|
|
application.make_installed
|
2018-01-03 20:45:57 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-24 05:09:25 -04:00
|
|
|
describe 'transition to updating' do
|
|
|
|
let(:project) { create(:project) }
|
|
|
|
let(:cluster) { create(:cluster, projects: [project]) }
|
|
|
|
|
|
|
|
subject { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
|
|
|
|
|
|
|
|
it 'sets last_update_started_at to now' do
|
|
|
|
Timecop.freeze do
|
2020-05-22 05:08:09 -04:00
|
|
|
expect { subject.make_updating }.to change { subject.reload.last_update_started_at }.to be_within(1.second).of(Time.current)
|
2020-03-24 05:09:25 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-04-12 01:42:48 -04:00
|
|
|
describe '#can_uninstall?' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus) }
|
|
|
|
|
|
|
|
subject { prometheus.can_uninstall? }
|
|
|
|
|
|
|
|
it { is_expected.to be_truthy }
|
|
|
|
end
|
|
|
|
|
2018-02-23 15:33:33 -05:00
|
|
|
describe '#prometheus_client' do
|
2019-12-05 22:08:02 -05:00
|
|
|
shared_examples 'exception caught for prometheus client' do
|
|
|
|
before do
|
|
|
|
allow(kube_client).to receive(:proxy_url).and_raise(exception)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns nil' do
|
|
|
|
expect(subject.prometheus_client).to be_nil
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-01-04 11:01:13 -05:00
|
|
|
context 'cluster is nil' do
|
|
|
|
it 'returns nil' do
|
|
|
|
expect(subject.cluster).to be_nil
|
2018-02-23 15:33:33 -05:00
|
|
|
expect(subject.prometheus_client).to be_nil
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context "cluster doesn't have kubeclient" do
|
|
|
|
let(:cluster) { create(:cluster) }
|
2019-12-18 19:08:01 -05:00
|
|
|
|
2018-01-04 11:01:13 -05:00
|
|
|
subject { create(:clusters_applications_prometheus, cluster: cluster) }
|
|
|
|
|
|
|
|
it 'returns nil' do
|
2018-02-23 15:33:33 -05:00
|
|
|
expect(subject.prometheus_client).to be_nil
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'cluster has kubeclient' do
|
2018-11-02 11:46:15 -04:00
|
|
|
let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
|
2018-09-06 06:03:38 -04:00
|
|
|
let(:kubernetes_url) { subject.cluster.platform_kubernetes.api_url }
|
|
|
|
let(:kube_client) { subject.cluster.kubeclient.core_client }
|
2018-01-04 11:01:13 -05:00
|
|
|
|
2018-11-02 11:46:15 -04:00
|
|
|
subject { create(:clusters_applications_prometheus, cluster: cluster) }
|
2018-01-04 11:01:13 -05:00
|
|
|
|
|
|
|
before do
|
2018-09-06 06:03:38 -04:00
|
|
|
subject.cluster.platform_kubernetes.namespace = 'a-namespace'
|
2018-11-02 11:46:15 -04:00
|
|
|
stub_kubeclient_discover(cluster.platform_kubernetes.api_url)
|
|
|
|
|
|
|
|
create(:cluster_kubernetes_namespace,
|
|
|
|
cluster: cluster,
|
|
|
|
cluster_project: cluster.cluster_project,
|
|
|
|
project: cluster.cluster_project.project)
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
|
|
|
|
2019-08-06 22:42:20 -04:00
|
|
|
it 'creates proxy prometheus_client' do
|
|
|
|
expect(subject.prometheus_client).to be_instance_of(Gitlab::PrometheusClient)
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
|
|
|
|
2019-08-06 22:42:20 -04:00
|
|
|
it 'copies proxy_url, options and headers from kube client to prometheus_client' do
|
|
|
|
expect(Gitlab::PrometheusClient)
|
|
|
|
.to(receive(:new))
|
|
|
|
.with(a_valid_url, kube_client.rest_client.options.merge(headers: kube_client.headers))
|
|
|
|
subject.prometheus_client
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
2018-05-17 02:47:48 -04:00
|
|
|
|
|
|
|
context 'when cluster is not reachable' do
|
2019-12-05 22:08:02 -05:00
|
|
|
it_behaves_like 'exception caught for prometheus client' do
|
|
|
|
let(:exception) { Kubeclient::HttpError.new(401, 'Unauthorized', nil) }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when there is a socket error while contacting cluster' do
|
|
|
|
it_behaves_like 'exception caught for prometheus client' do
|
|
|
|
let(:exception) { Errno::ECONNREFUSED }
|
2018-05-17 02:47:48 -04:00
|
|
|
end
|
|
|
|
|
2019-12-05 22:08:02 -05:00
|
|
|
it_behaves_like 'exception caught for prometheus client' do
|
|
|
|
let(:exception) { Errno::ECONNRESET }
|
2018-05-17 02:47:48 -04:00
|
|
|
end
|
|
|
|
end
|
2019-12-23 13:07:33 -05:00
|
|
|
|
|
|
|
context 'when the network is unreachable' do
|
|
|
|
it_behaves_like 'exception caught for prometheus client' do
|
|
|
|
let(:exception) { Errno::ENETUNREACH }
|
|
|
|
end
|
|
|
|
end
|
2018-01-04 11:01:13 -05:00
|
|
|
end
|
|
|
|
end
|
2018-03-01 18:46:02 -05:00
|
|
|
|
|
|
|
describe '#install_command' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus) }
|
|
|
|
|
2018-08-15 01:18:56 -04:00
|
|
|
subject { prometheus.install_command }
|
2018-03-01 18:46:02 -05:00
|
|
|
|
2018-08-15 01:18:56 -04:00
|
|
|
it { is_expected.to be_an_instance_of(Gitlab::Kubernetes::Helm::InstallCommand) }
|
2018-06-25 05:57:29 -04:00
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'is initialized with 3 arguments' do
|
2018-08-15 01:18:56 -04:00
|
|
|
expect(subject.name).to eq('prometheus')
|
|
|
|
expect(subject.chart).to eq('stable/prometheus')
|
2019-12-20 04:24:38 -05:00
|
|
|
expect(subject.version).to eq('9.5.2')
|
2019-01-03 08:41:53 -05:00
|
|
|
expect(subject).to be_rbac
|
2018-08-15 01:18:56 -04:00
|
|
|
expect(subject.files).to eq(prometheus.files)
|
2018-03-01 18:46:02 -05:00
|
|
|
end
|
2018-07-22 06:48:53 -04:00
|
|
|
|
2019-01-03 08:41:53 -05:00
|
|
|
context 'on a non rbac enabled cluster' do
|
2018-09-06 06:03:38 -04:00
|
|
|
before do
|
2019-01-03 08:41:53 -05:00
|
|
|
prometheus.cluster.platform_kubernetes.abac!
|
2018-09-06 06:03:38 -04:00
|
|
|
end
|
|
|
|
|
2019-01-03 08:41:53 -05:00
|
|
|
it { is_expected.not_to be_rbac }
|
2018-09-06 06:03:38 -04:00
|
|
|
end
|
|
|
|
|
2018-07-22 06:48:53 -04:00
|
|
|
context 'application failed to install previously' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus, :errored, version: '2.0.0') }
|
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'is initialized with the locked version' do
|
2019-12-20 04:24:38 -05:00
|
|
|
expect(subject.version).to eq('9.5.2')
|
2018-07-22 06:48:53 -04:00
|
|
|
end
|
|
|
|
end
|
2018-12-20 22:38:50 -05:00
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'does not install knative metrics' do
|
2019-08-02 15:45:43 -04:00
|
|
|
expect(subject.postinstall).to be_empty
|
2019-01-03 08:41:53 -05:00
|
|
|
end
|
|
|
|
|
2018-12-20 22:38:50 -05:00
|
|
|
context 'with knative installed' do
|
2019-02-15 06:16:45 -05:00
|
|
|
let(:knative) { create(:clusters_applications_knative, :updated ) }
|
2018-12-20 22:38:50 -05:00
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus, cluster: knative.cluster) }
|
|
|
|
|
|
|
|
subject { prometheus.install_command }
|
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'installs knative metrics' do
|
2018-12-20 22:38:50 -05:00
|
|
|
expect(subject.postinstall).to include("kubectl apply -f #{Clusters::Applications::Knative::METRICS_CONFIG}")
|
|
|
|
end
|
|
|
|
end
|
2018-03-01 18:46:02 -05:00
|
|
|
end
|
|
|
|
|
2019-04-12 07:15:50 -04:00
|
|
|
describe '#uninstall_command' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus) }
|
|
|
|
|
|
|
|
subject { prometheus.uninstall_command }
|
|
|
|
|
|
|
|
it { is_expected.to be_an_instance_of(Gitlab::Kubernetes::Helm::DeleteCommand) }
|
|
|
|
|
|
|
|
it 'has the application name' do
|
|
|
|
expect(subject.name).to eq('prometheus')
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'has files' do
|
|
|
|
expect(subject.files).to eq(prometheus.files)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'is rbac' do
|
|
|
|
expect(subject).to be_rbac
|
|
|
|
end
|
|
|
|
|
2019-10-08 05:06:09 -04:00
|
|
|
describe '#predelete' do
|
|
|
|
let(:knative) { create(:clusters_applications_knative, :updated ) }
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus, cluster: knative.cluster) }
|
|
|
|
|
|
|
|
subject { prometheus.uninstall_command.predelete }
|
|
|
|
|
|
|
|
it 'deletes knative metrics' do
|
|
|
|
metrics_config = Clusters::Applications::Knative::METRICS_CONFIG
|
|
|
|
is_expected.to include("kubectl delete -f #{metrics_config} --ignore-not-found")
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-04-12 07:15:50 -04:00
|
|
|
context 'on a non rbac enabled cluster' do
|
|
|
|
before do
|
|
|
|
prometheus.cluster.platform_kubernetes.abac!
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.not_to be_rbac }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-12-16 16:08:00 -05:00
|
|
|
describe '#patch_command' do
|
|
|
|
subject(:patch_command) { prometheus.patch_command(values) }
|
|
|
|
|
2018-12-19 03:50:12 -05:00
|
|
|
let(:prometheus) { build(:clusters_applications_prometheus) }
|
|
|
|
let(:values) { prometheus.values }
|
|
|
|
|
2019-12-16 16:08:00 -05:00
|
|
|
it { is_expected.to be_an_instance_of(::Gitlab::Kubernetes::Helm::PatchCommand) }
|
2018-12-19 03:50:12 -05:00
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'is initialized with 3 arguments' do
|
2019-12-16 16:08:00 -05:00
|
|
|
expect(patch_command.name).to eq('prometheus')
|
|
|
|
expect(patch_command.chart).to eq('stable/prometheus')
|
2019-12-20 04:24:38 -05:00
|
|
|
expect(patch_command.version).to eq('9.5.2')
|
2019-12-16 16:08:00 -05:00
|
|
|
expect(patch_command.files).to eq(prometheus.files)
|
2018-12-19 03:50:12 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#update_in_progress?' do
|
|
|
|
context 'when app is updating' do
|
|
|
|
it 'returns true' do
|
|
|
|
cluster = create(:cluster)
|
|
|
|
prometheus_app = build(:clusters_applications_prometheus, :updating, cluster: cluster)
|
|
|
|
|
|
|
|
expect(prometheus_app.update_in_progress?).to be true
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#update_errored?' do
|
|
|
|
context 'when app errored' do
|
|
|
|
it 'returns true' do
|
|
|
|
cluster = create(:cluster)
|
|
|
|
prometheus_app = build(:clusters_applications_prometheus, :update_errored, cluster: cluster)
|
|
|
|
|
|
|
|
expect(prometheus_app.update_errored?).to be true
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-08-07 08:39:38 -04:00
|
|
|
describe '#files' do
|
|
|
|
let(:application) { create(:clusters_applications_prometheus) }
|
|
|
|
let(:values) { subject[:'values.yaml'] }
|
|
|
|
|
|
|
|
subject { application.files }
|
|
|
|
|
2019-04-05 04:43:27 -04:00
|
|
|
it 'includes prometheus valid values' do
|
2018-08-07 08:39:38 -04:00
|
|
|
expect(values).to include('alertmanager')
|
|
|
|
expect(values).to include('kubeStateMetrics')
|
|
|
|
expect(values).to include('nodeExporter')
|
|
|
|
expect(values).to include('pushgateway')
|
|
|
|
expect(values).to include('serverFiles')
|
2018-03-01 18:46:02 -05:00
|
|
|
end
|
|
|
|
end
|
2018-12-19 03:50:12 -05:00
|
|
|
|
|
|
|
describe '#files_with_replaced_values' do
|
|
|
|
let(:application) { build(:clusters_applications_prometheus) }
|
|
|
|
let(:files) { application.files }
|
|
|
|
|
|
|
|
subject { application.files_with_replaced_values({ hello: :world }) }
|
|
|
|
|
|
|
|
it 'does not modify #files' do
|
2020-02-20 10:08:44 -05:00
|
|
|
expect(subject[:'values.yaml']).not_to eq(files[:'values.yaml'])
|
|
|
|
|
2018-12-19 03:50:12 -05:00
|
|
|
expect(files[:'values.yaml']).to eq(application.values)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns values.yaml with replaced values' do
|
|
|
|
expect(subject[:'values.yaml']).to eq({ hello: :world })
|
|
|
|
end
|
|
|
|
|
2020-02-20 10:08:44 -05:00
|
|
|
it 'uses values from #files, except for values.yaml' do
|
|
|
|
allow(application).to receive(:files).and_return({
|
|
|
|
'values.yaml': 'some value specific to files',
|
|
|
|
'file_a.txt': 'file_a',
|
|
|
|
'file_b.txt': 'file_b'
|
|
|
|
})
|
|
|
|
|
|
|
|
expect(subject.except(:'values.yaml')).to eq({
|
|
|
|
'file_a.txt': 'file_a',
|
|
|
|
'file_b.txt': 'file_b'
|
|
|
|
})
|
2018-12-19 03:50:12 -05:00
|
|
|
end
|
|
|
|
end
|
2019-12-05 22:08:02 -05:00
|
|
|
|
|
|
|
describe '#configured?' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
|
|
|
|
|
|
|
|
subject { prometheus.configured? }
|
|
|
|
|
|
|
|
context 'when a kubenetes client is present' do
|
|
|
|
let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
|
|
|
|
|
|
|
|
it { is_expected.to be_truthy }
|
|
|
|
|
|
|
|
context 'when it is not availalble' do
|
|
|
|
let(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
|
|
|
|
|
|
|
|
it { is_expected.to be_falsey }
|
|
|
|
end
|
2020-02-11 01:09:46 -05:00
|
|
|
|
|
|
|
context 'when the kubernetes URL is blocked' do
|
|
|
|
before do
|
|
|
|
blocked_ip = '127.0.0.1' # localhost addresses are blocked by default
|
|
|
|
|
|
|
|
stub_all_dns(cluster.platform.api_url, ip_address: blocked_ip)
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.to be_falsey }
|
|
|
|
end
|
2019-12-05 22:08:02 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'when a kubenetes client is not present' do
|
|
|
|
let(:cluster) { create(:cluster) }
|
|
|
|
|
|
|
|
it { is_expected.to be_falsy }
|
|
|
|
end
|
|
|
|
end
|
2020-03-17 17:09:16 -04:00
|
|
|
|
2020-03-24 05:09:25 -04:00
|
|
|
describe '#updated_since?' do
|
|
|
|
let(:cluster) { create(:cluster) }
|
|
|
|
let(:prometheus_app) { build(:clusters_applications_prometheus, cluster: cluster) }
|
2020-05-22 05:08:09 -04:00
|
|
|
let(:timestamp) { Time.current - 5.minutes }
|
2020-03-24 05:09:25 -04:00
|
|
|
|
|
|
|
around do |example|
|
|
|
|
Timecop.freeze { example.run }
|
|
|
|
end
|
|
|
|
|
|
|
|
before do
|
2020-05-22 05:08:09 -04:00
|
|
|
prometheus_app.last_update_started_at = Time.current
|
2020-03-24 05:09:25 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'when app does not have status failed' do
|
|
|
|
it 'returns true when last update started after the timestamp' do
|
|
|
|
expect(prometheus_app.updated_since?(timestamp)).to be true
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns false when last update started before the timestamp' do
|
2020-05-22 05:08:09 -04:00
|
|
|
expect(prometheus_app.updated_since?(Time.current + 5.minutes)).to be false
|
2020-03-24 05:09:25 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when app has status failed' do
|
|
|
|
it 'returns false when last update started after the timestamp' do
|
|
|
|
prometheus_app.status = 6
|
|
|
|
|
|
|
|
expect(prometheus_app.updated_since?(timestamp)).to be false
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-17 17:09:16 -04:00
|
|
|
describe 'alert manager token' do
|
|
|
|
subject { create(:clusters_applications_prometheus) }
|
|
|
|
|
|
|
|
context 'when not set' do
|
|
|
|
it 'is empty by default' do
|
|
|
|
expect(subject.alert_manager_token).to be_nil
|
|
|
|
expect(subject.encrypted_alert_manager_token).to be_nil
|
|
|
|
expect(subject.encrypted_alert_manager_token_iv).to be_nil
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#generate_alert_manager_token!' do
|
|
|
|
it 'generates a token' do
|
|
|
|
subject.generate_alert_manager_token!
|
|
|
|
|
|
|
|
expect(subject.alert_manager_token).to match(/\A\h{32}\z/)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when set' do
|
|
|
|
let(:token) { SecureRandom.hex }
|
|
|
|
|
|
|
|
before do
|
|
|
|
subject.update!(alert_manager_token: token)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'reads the token' do
|
|
|
|
expect(subject.alert_manager_token).to eq(token)
|
|
|
|
expect(subject.encrypted_alert_manager_token).not_to be_nil
|
|
|
|
expect(subject.encrypted_alert_manager_token_iv).not_to be_nil
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#generate_alert_manager_token!' do
|
|
|
|
it 'does not re-generate the token' do
|
|
|
|
subject.generate_alert_manager_token!
|
|
|
|
|
|
|
|
expect(subject.alert_manager_token).to eq(token)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2017-12-22 12:23:43 -05:00
|
|
|
end
|