Skip to content

Ceilometer extractor now extends nova information. #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 4, 2015
Merged

Ceilometer extractor now extends nova information. #21

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 28 additions & 122 deletions caso/extract/ceilometer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,14 @@
# License for the specific language governing permissions and limitations
# under the License.

import datetime

import ceilometerclient.client
import dateutil.parser
from dateutil import tz
import glanceclient.client
from oslo.config import cfg

from caso.extract import base
from caso.extract import nova
from caso import log
from caso import record

CONF = cfg.CONF
CONF.import_opt("site_name", "caso.extract.manager")
CONF.import_opt("user", "caso.extract.base", "extractor")
CONF.import_opt("password", "caso.extract.base", "extractor")
CONF.import_opt("endpoint", "caso.extract.base", "extractor")
Expand All @@ -36,7 +30,7 @@
LOG = log.getLogger(__name__)


class CeilometerExtractor(base.BaseExtractor):
class CeilometerExtractor(nova.OpenStackExtractor):
def _get_ceilometer_client(self, tenant):
return ceilometerclient.client.get_client(
'2',
Expand All @@ -46,18 +40,6 @@ def _get_ceilometer_client(self, tenant):
os_tenant_name=tenant,
insecure=CONF.extractor.insecure)

def _get_glance_client(self, ks_client):
glance_ep = ks_client.service_catalog.get_endpoints(
service_type='image',
endpoint_type='public')
glance_url = glance_ep['image'][0]['publicURL']
# glance client does not seem to work with user/password
return glanceclient.client.Client(
'2',
glance_url,
token=ks_client.auth_token,
insecure=CONF.extractor.insecure)

def _build_query(self, project_id=None, start_date=None, end_date=None):
q = []
if project_id:
Expand All @@ -80,19 +62,24 @@ def _fill_metric(self, metric_name, samples, records,
"""

instance_timestamps = {}
the_past = datetime.datetime(1, 1, 1)
for sample in samples:
instance_id = get_id(sample)
r = records.get(instance_id)
if not r:
# XXX: there is a sample for a VM that has no instance sample?
LOG.debug("VM with some usage info but no instance metric?!")
try:
r = records[instance_id]
except KeyError:
continue
instance_ts = instance_timestamps.get(instance_id, the_past)
# takes the maximum value from ceilometer
sample_value = unit_conv(sample.counter_volume)
instance_ts = instance_timestamps.get(instance_id, None)
if instance_ts is None:
r.__dict__[metric_name] = sample_value
try:
r.__dict__[metric_name] = max(r.__dict__[metric_name],
sample_value)
except KeyError:
r.__dict__[metric_name] = sample_value
sample_ts = dateutil.parser.parse(sample.timestamp)
if sample_ts > instance_ts:
r.__dict__[metric_name] = unit_conv(sample.counter_volume)
instance_timestamps[instance_id] = sample_ts
instance_timestamps[instance_id] = sample_ts

def _fill_cpu_metric(self, cpu_samples, records):
self._fill_metric('cpu_duration', cpu_samples, records,
Expand All @@ -105,104 +92,23 @@ def _fill_net_metric(self, metric_name, net_samples, records):
# convert bytes to GB
unit_conv=lambda v: int(v / 2 ** 30))

def _get_time_field(self, metadata, fields=[]):
for f in fields:
t = metadata.get(f, None)
if t:
dateutil.parser.parse(t).replace(tzinfo=None)
else:
return None

def _build_record(self, instance, users, vo, images, now):
metadata = instance.resource_metadata
status = self.vm_status(metadata.get('state', ''))
instance_image_id = metadata.get('image.id')
if not instance_image_id:
instance_image_id = metadata.get('image_meta.base_image_ref')
image_id = None
for image in images:
if instance_image_id == image.id:
image_id = image.get('vmcatcher_event_ad_mpuri', None)
break
else:
image_id = instance_image_id
r = record.CloudRecord(instance.resource_id,
CONF.site_name,
metadata.get('display_name'),
instance.user_id,
instance.project_id,
vo,
cpu_count=metadata.get('vcpus'),
memory=metadata.get('memory_gb'),
disk=metadata.get('disk_gb'),
# this should contain "caso ceilometer vX"
cloud_type="OpenStack",
status=status,
image_id=image_id,
user_dn=users.get(instance.user_id, None))

start_time = self._get_time_field(metadata,
('launched_at', 'created_at'))
end_time = self._get_time_field(metadata,
('terminated_at', 'deleted_at'))
if start_time:
r.start_time = int(start_time.strftime("%s"))
if end_time:
r.end_time = int(end_time.strftime("%s"))
wall = end_time - start_time
r.wall_duration = int(wall.total_seconds())
else:
wall = now - start_time
r.wall_duration = int(wall.total_seconds())
return r

def extract_for_tenant(self, tenant, lastrun):
now = datetime.datetime.now(tz.tzutc())
now.replace(tzinfo=None)

records = super(CeilometerExtractor,
self).extract_for_tenant(tenant, lastrun)
# Try and except here
# Getting clients
ks_client = self._get_keystone_client(tenant)
client = self._get_ceilometer_client(tenant)
glance_client = self._get_glance_client(ks_client)

# users
users = self._get_keystone_users(ks_client)
tenant_id = ks_client.tenant_id

search_query = self._build_query(tenant_id, lastrun)
instances = client.samples.list('instance', search_query)

# XXX should we query glance for every VM or not?
images = list(glance_client.images.list())

records = {}

vo = self.voms_map.get(tenant)

for instance in instances:
# it seems the only event type with relevant metadata is
# 'compute.instance.exists'
ev_type = instance.resource_metadata.get('event_type', None)
if ev_type != 'compute.instance.exists':
continue
# this assumes that records are returned with decreasing timestamp
# so the status and dates are the last ones.
if instance.resource_id in records:
continue
records[instance.resource_id] = self._build_record(instance,
users,
vo,
images,
now)
ks_conn = self._get_keystone_client(tenant)
conn = self._get_ceilometer_client(tenant)
# See comment in nova.py, remove TZ from the dates.
lastrun = lastrun.replace(tzinfo=None)
search_query = self._build_query(ks_conn.tenant_id, lastrun)

cpu = client.samples.list(meter_name='cpu', q=search_query)
cpu = conn.samples.list(meter_name='cpu', q=search_query)
self._fill_cpu_metric(cpu, records)
net_in = client.samples.list(meter_name='network.incoming.bytes',
q=search_query)
net_in = conn.samples.list(meter_name='network.incoming.bytes',
q=search_query)
self._fill_net_metric('network_in', net_in, records)
net_out = client.samples.list(meter_name='network.outcoming.bytes',
q=search_query)
net_out = conn.samples.list(meter_name='network.outcoming.bytes',
q=search_query)
self._fill_net_metric('network_out', net_out, records)

return records