#!/usr/bin/env python
# -*- test-case-name: calendarserver.tools.test.test_calverify -*-
##
# Copyright (c) 2011-2017 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##


"""
This tool scans the calendar store to analyze organizer/attendee event
states to verify that the organizer's view of attendee state matches up
with the attendees' views. It can optionally apply a fix to bring the two
views back into line.

In theory the implicit scheduling model should eliminate the possibility
of mismatches, however, because we store separate resources for organizer
and attendee events, there is a possibility of mismatch. This is greatly
lessened via the new transaction model of database changes, but it is
possible there are edge cases or actual implicit processing errors we have
missed. This tool will allow us to track mismatches to help determine these
errors and get them fixed.

Even in the long term if we move to a "single instance" store where the
organizer event resource is the only one we store (with attendee views
derived from that), in a situation where we have server-to-server scheduling
it is possible for mismatches to creep in. In that case having a way to analyze
multiple DBs for inconsistency would be good too.

"""

from __future__ import print_function

from uuid import uuid4
import collections
import itertools
import sys
import time
import traceback

from calendarserver.tools import tables
from calendarserver.tools.cmdline import utilityMain, WorkerService
from pycalendar.datetime import DateTime
from pycalendar.exceptions import ErrorBase
from pycalendar.icalendar import definitions
from pycalendar.icalendar.calendar import Calendar
from pycalendar.period import Period
from pycalendar.timezone import Timezone
from twext.enterprise.dal.syntax import Select, Parameter, Count, Update
from twext.python.log import Logger
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python import usage
from twisted.python.usage import Options
from twistedcaldav.datafilters.peruserdata import PerUserDataFilter
from twistedcaldav.dateops import pyCalendarToSQLTimestamp
from twistedcaldav.ical import Component, InvalidICalendarDataError, Property, PERUSER_COMPONENT
from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
from twistedcaldav.timezones import TimezoneCache
from txdav.caldav.datastore.scheduling.icalsplitter import iCalSplitter
from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
from txdav.caldav.datastore.scheduling.itip import iTipGenerator
from txdav.caldav.datastore.sql import CalendarStoreFeatures
from txdav.caldav.datastore.util import normalizationLookup
from txdav.caldav.icalendarstore import ComponentUpdateState
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
from txdav.common.icommondatastore import InternalDataStoreError
from txdav.who.idirectory import RecordType as CalRecordType, AutoScheduleMode


log = Logger()


# Monkey patch
def new_validRecurrenceIDs(self, doFix=True):

    fixed = []
    unfixed = []

    # Detect invalid occurrences and fix by adding RDATEs for them
    master = self.masterComponent()
    if master is not None:
        # Get the set of all recurrence IDs
        all_rids = set(self.getComponentInstances())
        if None in all_rids:
            all_rids.remove(None)

        # If the master has no recurrence properties treat any other components as invalid
        if master.isRecurring():

            # Remove all EXDATEs with a matching RECURRENCE-ID. Do this before we start
            # processing of valid instances just in case the matching R-ID is also not valid and
            # thus will need RDATE added.
            exdates = {}
            for property in list(master.properties("EXDATE")):
                for exdate in property.value():
                    exdates[exdate.getValue()] = property
            for rid in all_rids:
                if rid in exdates:
                    if doFix:
                        property = exdates[rid]
                        for value in property.value():
                            if value.getValue() == rid:
                                property.value().remove(value)
                                break
                        master.removeProperty(property)
                        if len(property.value()) > 0:
                            master.addProperty(property)
                        del exdates[rid]
                        fixed.append("Removed EXDATE for valid override: %s" % (rid,))
                    else:
                        unfixed.append("EXDATE for valid override: %s" % (rid,))

            # Get the set of all valid recurrence IDs
            valid_rids = self.validInstances(all_rids, ignoreInvalidInstances=True)

            # Get the set of all RDATEs and add those to the valid set
            rdates = []
            for property in master.properties("RDATE"):
                rdates.extend([_rdate.getValue() for _rdate in property.value()])
            valid_rids.update(set(rdates))

            # Remove EXDATEs predating master
            dtstart = master.propertyValue("DTSTART")
            if dtstart is not None:
                for property in list(master.properties("EXDATE")):
                    newValues = []
                    changed = False
                    for exdate in property.value():
                        exdateValue = exdate.getValue()
                        if exdateValue < dtstart:
                            if doFix:
                                fixed.append("Removed earlier EXDATE: %s" % (exdateValue,))
                            else:
                                unfixed.append("EXDATE earlier than master: %s" % (exdateValue,))
                            changed = True
                        else:
                            newValues.append(exdateValue)

                    if changed and doFix:
                        # Remove the property...
                        master.removeProperty(property)
                        if newValues:
                            # ...and add it back only if it still has values
                            property.setValue(newValues)
                            master.addProperty(property)

        else:
            valid_rids = set()

        # Determine the invalid recurrence IDs by set subtraction
        invalid_rids = all_rids - valid_rids

        # Add RDATEs for the invalid ones, or remove any EXDATE.
        for invalid_rid in invalid_rids:
            brokenComponent = self.overriddenComponent(invalid_rid)
            brokenRID = brokenComponent.propertyValue("RECURRENCE-ID")
            if doFix:
                master.addProperty(Property("RDATE", [brokenRID, ]))
                fixed.append(
                    "Added RDATE for invalid occurrence: %s" %
                    (brokenRID,))
            else:
                unfixed.append("Invalid occurrence: %s" % (brokenRID,))

    return fixed, unfixed


def new_hasDuplicateAlarms(self, doFix=False):
    """
    test and optionally remove alarms that have the same ACTION and TRIGGER values in the same component.
    """
    changed = False
    if self.name() in ("VCALENDAR", PERUSER_COMPONENT,):
        for component in self.subcomponents():
            if component.name() in ("VTIMEZONE",):
                continue
            changed = component.hasDuplicateAlarms(doFix) or changed
    else:
        action_trigger = set()
        for component in tuple(self.subcomponents()):
            if component.name() == "VALARM":
                item = (component.propertyValue("ACTION"), component.propertyValue("TRIGGER"),)
                if item in action_trigger:
                    if doFix:
                        self.removeComponent(component)
                    changed = True
                else:
                    action_trigger.add(item)
    return changed

Component.validRecurrenceIDs = new_validRecurrenceIDs
if not hasattr(Component, "maxAlarmCounts"):
    Component.hasDuplicateAlarms = new_hasDuplicateAlarms

VERSION = "13"


def printusage(e=None):
    if e:
        print(e)
        print("")
    try:
        CalVerifyOptions().opt_help()
    except SystemExit:
        pass
    if e:
        sys.exit(64)
    else:
        sys.exit(0)


description = """
Usage: calendarserver_verify_data [options]
Version: %s

This tool scans the calendar store to look for and correct any
problems.

OPTIONS:

Modes of operation:

-h                  : print help and exit.
--ical              : verify iCalendar data.
--mismatch          : verify scheduling state.
--missing           : display orphaned calendar homes - can be used.
                      with either --ical or --mismatch.
--double            : detect double-bookings.
--dark-purge        : purge room/resource events with invalid organizer
--split             : split recurring event
--upgrade           : upgrade iCalendar data.

--nuke PATH|UID|RID : remove specific calendar resources - can
                      only be used by itself. PATH is the full
                      /calendars/__uids__/XXX/YYY/ZZZ.ics object
                      resource path. UID is the iCalendar UID,
                      prefixed with "uid:", of the resources to
                      remove. RID is the SQL DB resource-id.

Options for all modes:

--fix      : changes are only made when this is present.
--config   : caldavd.plist file for the server.
-v         : verbose logging

Options for --ical / --upgrade:

--uuid     : only scan specified calendar homes. Can be a partial GUID
             to scan all GUIDs with that as a prefix.
--uid      : scan only calendar data with the specific iCalendar UID.
--calendar : When scanning calendar homes, only scan calendars with the
             specified calendar resource name.
--path     : Scan the calendar home or calendar identified by the
             specified URI

Options for --mismatch:

--uid      : look for mismatches with the specified iCalendar UID only.
--details  : log extended details on each mismatch.
--tzid     : timezone to adjust details to.

Options for --double:

--uuid     : only scan specified calendar homes. Can be a partial GUID
             to scan all GUIDs with that as a prefix or "*" for all GUIDS
             (that are marked as resources or locations in the directory).
--tzid     : timezone to adjust details to.
--summary  : report only which GUIDs have double-bookings - no details.
--days     : number of days ahead to scan [DEFAULT: 365]

Options for --double:
If none of (--no-organizer, --invalid-organizer, --disabled-organizer) is present, it
will default to (--invalid-organizer, --disabled-organizer).

Options for --dark-purge:

--uuid     : only scan specified calendar homes. Can be a partial GUID
             to scan all GUIDs with that as a prefix or "*" for all GUIDS
             (that are marked as resources or locations in the directory).
--summary  : report only which GUIDs have double-bookings - no details.
--no-organizer       : only detect events without an organizer
--invalid-organizer  : only detect events with an organizer not in the directory
--disabled-organizer : only detect events with an organizer disabled for calendaring

Options for --missing-location:

--uuid     : only scan specified calendar homes. Can be a partial GUID
             to scan all GUIDs with that as a prefix or "*" for all GUIDS
             (that are marked as locations in the directory).
--summary  : report only which GUIDs have bad events - no details.

Options for --split:

--path     : URI path to resource to split.
--rid      : UTC date-time where split occurs (YYYYMMDDTHHMMSSZ).
--summary  : only print a list of recurrences in the resource - no splitting.

CHANGES
v8: Detects ORGANIZER or ATTENDEE properties with mailto: calendar user
    addresses for users that have valid directory records. Fix is to
    replace the value with a urn:x-uid: form.

v9: Detects double-bookings.

v10: Purges data for invalid users.

v11: Allows manual splitting of recurring events.

v12: Fix double-booking false positives caused by timezones-by-reference.

v13: Add new options for --nuke and --ical. Add fix for invalid GEO.

""" % (VERSION,)


def safePercent(x, y, multiplier=100.0):
    return ((multiplier * x) / y) if y else 0


class CalVerifyOptions(Options):
    """
    Command-line options for 'calendarserver_verify_data'
    """

    synopsis = description

    optFlags = [
        ['ical', 'i', "Calendar data check."],
        ['debug', 'D', "Debug logging."],
        ['mismatch', 's', "Detect organizer/attendee mismatches."],
        ['missing', 'm', "Show 'orphaned' homes."],
        ['double', 'd', "Detect double-bookings."],
        ['dark-purge', 'p', "Purge room/resource events with invalid organizer."],
        ['missing-location', 'g', "Room events with missing location."],
        ['split', 'l', "Split an event."],
        ['upgrade', 'u', "Upgrade calendar data."],

        ['fix', 'x', "Fix problems."],
        ['verbose', 'v', "Verbose logging."],
        ['details', 'V', "Detailed logging."],
        ['summary', 'S', "Summary of double-bookings/split."],
        ['tzid', 't', "Timezone to adjust displayed times to."],

        ['no-organizer', '', "Detect dark events without an organizer"],
        ['invalid-organizer', '', "Detect dark events with an organizer not in the directory"],
        ['disabled-organizer', '', "Detect dark events with a disabled organizer"],
    ]

    optParameters = [
        ['config', 'f', DEFAULT_CONFIG_FILE, "Specify caldavd.plist configuration path."],
        ['uuid', 'u', "", "Only check this user."],
        ['uid', 'U', "", "Only this event UID."],
        ['calendar', '', "", "Only this calendar resource name."],
        ['nuke', 'e', "", "Remove event given its path."],
        ['days', 'T', "365", "Number of days for scanning events into the future."],
        ['path', '', "", "Split event given its path."],
        ['rid', '', "", "Split date-time."],
    ]

    def __init__(self):
        super(CalVerifyOptions, self).__init__()
        self.outputName = '-'

    def getUsage(self, width=None):
        return ""

    def opt_output(self, filename):
        """
        Specify output file path (default: '-', meaning stdout).
        """
        self.outputName = filename

    opt_o = opt_output

    def openOutput(self):
        """
        Open the appropriate output file based on the '--output' option.
        """
        if self.outputName == '-':
            return sys.stdout
        else:
            return open(self.outputName, 'wb')


class CalVerifyService(WorkerService, object):
    """
    Base class for common service behaviors.
    """

    def __init__(self, store, options, output, reactor, config):
        super(CalVerifyService, self).__init__(store)
        self.options = options
        self.output = output
        self.reactor = reactor
        self.config = config
        self._directory = store.directoryService()
        self._principalCollection = self.rootResource().getChild("principals")

        self.cuaCache = {}

        self.results = {}
        self.summary = []
        self.total = 0
        self.totalErrors = None
        self.totalExceptions = None

        TimezoneCache.create()

    def title(self):
        return ""

    @inlineCallbacks
    def doWork(self):
        """
        Do the operation stopping the reactor when done.
        """
        self.output.write("\n---- CalVerify %s version: %s ----\n" % (self.title(), VERSION,))

        try:
            yield self.doAction()
        except Exception as e:
            self.output.write("CalVerify Failure: %s\n" % (str(e),))
            log.failure("CalVerify Failure")
        except:
            self.output.write("CalVerify Failure: unknown exception\n")
            log.failure("CalVerify Failure")
        finally:
            self.output.close()

    def directoryService(self):
        """
        Return the directory service
        """
        return self._directory

    @inlineCallbacks
    def getAllHomeUIDs(self):
        ch = schema.CALENDAR_HOME
        rows = (yield Select(
            [ch.OWNER_UID, ],
            From=ch,
        ).on(self.txn))
        returnValue(tuple([uid[0] for uid in rows]))

    @inlineCallbacks
    def getMatchingHomeUIDs(self, uuid):
        ch = schema.CALENDAR_HOME
        kwds = {"uuid": uuid}
        rows = (yield Select(
            [ch.OWNER_UID, ],
            From=ch,
            Where=(ch.OWNER_UID.StartsWith(Parameter("uuid"))),
        ).on(self.txn, **kwds))
        returnValue(tuple([uid[0] for uid in rows]))

    @inlineCallbacks
    def countHomeContents(self, uid):
        ch = schema.CALENDAR_HOME
        cb = schema.CALENDAR_BIND
        co = schema.CALENDAR_OBJECT
        kwds = {"UID": uid}
        rows = (yield Select(
            [Count(co.RESOURCE_ID), ],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID)),
            Where=(ch.OWNER_UID == Parameter("UID"))
        ).on(self.txn, **kwds))
        returnValue(int(rows[0][0]) if rows else 0)

    @inlineCallbacks
    def getAllResourceInfo(self, inbox=False):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME

        if inbox:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN)
        else:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN).And(
                cb.CALENDAR_RESOURCE_NAME != "inbox")

        kwds = {}
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=cojoin),
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoWithUUID(self, uuid, inbox=False, calendar=None):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME

        if calendar:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN).And(
                cb.CALENDAR_RESOURCE_NAME == calendar)
        elif inbox:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN)
        else:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN).And(
                cb.CALENDAR_RESOURCE_NAME != "inbox")

        kwds = {"uuid": uuid}
        if len(uuid) != 36:
            where = (ch.OWNER_UID.StartsWith(Parameter("uuid")))
        else:
            where = (ch.OWNER_UID == Parameter("uuid"))
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=cojoin),
            Where=where,
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoTimeRange(self, start):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME
        tr = schema.TIME_RANGE
        kwds = {
            "Start": pyCalendarToSQLTimestamp(start),
            "Max": pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0))
        }
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN).And(
                    cb.CALENDAR_RESOURCE_NAME != "inbox").And(
                    co.ORGANIZER != "")).join(
                tr, type="left", on=(co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID)),
            Where=(tr.START_DATE >= Parameter("Start")).Or(co.RECURRANCE_MAX <= Parameter("Start")),
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoWithUID(self, uid, inbox=False):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME

        if inbox:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN)
        else:
            cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                cb.BIND_MODE == _BIND_MODE_OWN).And(
                cb.CALENDAR_RESOURCE_NAME != "inbox")

        kwds = {
            "UID": uid,
        }
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=cojoin),
            Where=(co.ICALENDAR_UID == Parameter("UID")),
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoTimeRangeWithUUID(self, start, uuid):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME
        tr = schema.TIME_RANGE
        kwds = {
            "Start": pyCalendarToSQLTimestamp(start),
            "Max": pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0)),
            "UUID": uuid,
        }
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN).And(
                    cb.CALENDAR_RESOURCE_NAME != "inbox")).join(
                tr, type="left", on=(co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID)),
            Where=(ch.OWNER_UID == Parameter("UUID")).And((tr.START_DATE >= Parameter("Start")).Or(co.RECURRANCE_MAX <= Parameter("Start"))),
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoTimeRangeWithUUIDForAllUID(self, start, uuid):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME
        tr = schema.TIME_RANGE

        cojoin = (cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
            cb.BIND_MODE == _BIND_MODE_OWN).And(
            cb.CALENDAR_RESOURCE_NAME != "inbox")

        kwds = {
            "Start": pyCalendarToSQLTimestamp(start),
            "Max": pyCalendarToSQLTimestamp(DateTime(1900, 1, 1, 0, 0, 0)),
            "UUID": uuid,
        }
        rows = (yield Select(
            [ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=cojoin),
            Where=(co.ICALENDAR_UID.In(Select(
                [co.ICALENDAR_UID],
                From=ch.join(
                    cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                    co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                        cb.BIND_MODE == _BIND_MODE_OWN).And(
                        cb.CALENDAR_RESOURCE_NAME != "inbox").And(
                        co.ORGANIZER != "")).join(
                    tr, type="left", on=(co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID)),
                Where=(ch.OWNER_UID == Parameter("UUID")).And((tr.START_DATE >= Parameter("Start")).Or(co.RECURRANCE_MAX <= Parameter("Start"))),
                GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
            ))),
            GroupBy=(ch.OWNER_UID, co.RESOURCE_ID, co.ICALENDAR_UID, cb.CALENDAR_RESOURCE_NAME, co.MD5, co.ORGANIZER, co.CREATED, co.MODIFIED,),
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def getAllResourceInfoForResourceID(self, resid):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME
        kwds = {"resid": resid}
        rows = (yield Select(
            [ch.RESOURCE_ID, cb.CALENDAR_RESOURCE_ID, ],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN)),
            Where=(co.RESOURCE_ID == Parameter("resid")),
        ).on(self.txn, **kwds))
        returnValue(rows[0])

    @inlineCallbacks
    def getResourceID(self, home, calendar, resource):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME

        kwds = {
            "home": home,
            "calendar": calendar,
            "resource": resource,
        }
        rows = (yield Select(
            [co.RESOURCE_ID],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID)),
            Where=(ch.OWNER_UID == Parameter("home")).And(
                cb.CALENDAR_RESOURCE_NAME == Parameter("calendar")).And(
                co.RESOURCE_NAME == Parameter("resource")
            ),
        ).on(self.txn, **kwds))
        returnValue(rows[0][0] if rows else None)

    @inlineCallbacks
    def getCalendar(self, resid, doFix=False):
        self.parseError = None
        self.parseException = None
        co = schema.CALENDAR_OBJECT
        kwds = {"ResourceID": resid}
        rows = (yield Select(
            [co.ICALENDAR_TEXT],
            From=co,
            Where=(
                co.RESOURCE_ID == Parameter("ResourceID")
            ),
        ).on(self.txn, **kwds))
        try:
            caldata = Calendar.parseText(rows[0][0]) if rows else None
        except ErrorBase as e:
            self.parseError = "Failed to parse"
            self.parseException = str(e)
            returnValue(None)

        returnValue(caldata)

    @inlineCallbacks
    def getCalendarForOwnerByUID(self, owner, uid):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME

        kwds = {"OWNER": owner, "UID": uid}
        rows = (yield Select(
            [co.ICALENDAR_TEXT, co.RESOURCE_ID, co.CREATED, co.MODIFIED, ],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN).And(
                    cb.CALENDAR_RESOURCE_NAME != "inbox")),
            Where=(ch.OWNER_UID == Parameter("OWNER")).And(co.ICALENDAR_UID == Parameter("UID")),
        ).on(self.txn, **kwds))

        try:
            caldata = Calendar.parseText(rows[0][0]) if rows else None
        except ErrorBase:
            returnValue((None, None, None, None,))

        returnValue((caldata, rows[0][1], rows[0][2], rows[0][3],) if rows else (None, None, None, None,))

    @inlineCallbacks
    def removeEvent(self, resid):
        """
        Remove the calendar resource specified by resid - this is a force remove - no implicit
        scheduling is required so we use store apis directly.
        """

        try:
            homeID, calendarID = yield self.getAllResourceInfoForResourceID(resid)
            home = yield self.txn.calendarHomeWithResourceID(homeID)
            calendar = yield home.childWithID(calendarID)
            calendarObj = yield calendar.objectResourceWithID(resid)
            objname = calendarObj.name()
            yield calendarObj.purge(implicitly=False)
            yield self.txn.commit()
            self.txn = self.store.newTransaction()

            self.results.setdefault("Fix remove", set()).add((home.name(), calendar.name(), objname,))

            returnValue(True)
        except Exception, e:
            print("Failed to remove resource whilst fixing: %d\n%s" % (resid, e,))
            returnValue(False)

    def logResult(self, key, value, total=None):
        self.output.write("%s: %s\n" % (key, value,))
        self.results[key] = value
        self.addToSummary(key, value, total)

    def addToSummary(self, title, count, total=None):
        if total is not None:
            percent = safePercent(count, total),
        else:
            percent = ""
        self.summary.append((title, count, percent))

    def addSummaryBreak(self):
        self.summary.append(None)

    def printSummary(self):
        # Print summary of results
        table = tables.Table()
        table.addHeader(("Item", "Count", "%"))
        table.setDefaultColumnFormats(
            (
                tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
                tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
                tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
            )
        )
        for item in self.summary:
            table.addRow(item)

        if self.totalErrors is not None:
            table.addRow(None)
            table.addRow(("Total Errors", self.totalErrors, safePercent(self.totalErrors, self.total),))

        self.output.write("\n")
        self.output.write("Overall Summary:\n")
        table.printTable(os=self.output)


class NukeService(CalVerifyService):
    """
    Service which removes specific events.
    """

    def title(self):
        return "Nuke Service"

    @inlineCallbacks
    def doAction(self):
        """
        Remove a resource using either its path or resource id. When doing this do not
        read the iCalendar data which may be corrupt.
        """

        self.output.write("\n---- Removing calendar resource ----\n")
        self.txn = self.store.newTransaction()

        nuke = self.options["nuke"]
        if nuke.startswith("/calendars/__uids__/"):
            pathbits = nuke.split("/")
            if len(pathbits) != 6:
                printusage("Not a valid calendar object resource path: %s" % (nuke,))
            homeName = pathbits[3]
            calendarName = pathbits[4]
            resourceName = pathbits[5]

            rid = yield self.getResourceID(homeName, calendarName, resourceName)
            if rid is None:
                yield self.txn.commit()
                self.txn = None
                self.output.write("\n")
                self.output.write("Path does not exist. Nothing nuked.\n")
                returnValue(None)
            rids = [(int(rid), homeName, calendarName,)]
        elif nuke.startswith("uid:"):
            uid = nuke[4:]
            rows = yield self.getAllResourceInfoWithUID(uid, inbox=True)
            rids = []
            for owner, resid, _ignore_uid, calname, _ignore_md5, _ignore_organizer, _ignore_created, _ignore_modified in rows:
                rids.append((resid, owner, calname,))
        else:
            try:
                rids = [(int(nuke), "-", "-",)]
            except ValueError:
                printusage("nuke argument must be a calendar object path or an SQL resource-id")

        self.output.write("\nNumber of resources: %d\n\n" % (len(rids),))
        for rid, owner, calname in rids:
            if self.options["fix"]:
                result = yield self.removeEvent(rid)
                if result:
                    self.output.write("Removed resource: %s. Owner: %s. Calendar: %s\n" % (rid, owner, calname))
            else:
                self.output.write("Resource: %s. Owner: %s. Calendar: %s\n" % (rid, owner, calname))
        yield self.txn.commit()
        self.txn = None


class OrphansService(CalVerifyService):
    """
    Service which detects orphaned calendar homes.
    """

    def title(self):
        return "Orphans Service"

    @inlineCallbacks
    def doAction(self):
        """
        Report on home collections for which there are no directory records, or record is for user on
        a different pod, or a user not enabled for calendaring.
        """
        self.output.write("\n---- Finding calendar homes with missing or disabled directory records ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()
        uids = yield self.getAllHomeUIDs()
        if self.options["verbose"]:
            self.output.write("getAllHomeUIDs time: %.1fs\n" % (time.time() - t,))
        missing = []
        wrong_server = []
        disabled = []
        uids_len = len(uids)
        uids_div = 1 if uids_len < 100 else uids_len / 100
        self.addToSummary("Total Homes", uids_len)

        for ctr, uid in enumerate(uids):
            if self.options["verbose"] and divmod(ctr, uids_div)[1] == 0:
                self.output.write(("\r%d of %d (%d%%)" % (
                    ctr + 1,
                    uids_len,
                    ((ctr + 1) * 100 / uids_len),
                )).ljust(80))
                self.output.flush()

            record = yield self.directoryService().recordWithUID(uid)
            if record is None:
                contents = yield self.countHomeContents(uid)
                missing.append((uid, contents,))
            elif not record.thisServer():
                contents = yield self.countHomeContents(uid)
                wrong_server.append((uid, contents,))
            elif not record.hasCalendars:
                contents = yield self.countHomeContents(uid)
                disabled.append((uid, contents,))

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(ctr, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"]:
            self.output.write("\r".ljust(80) + "\n")

        # Print table of results
        table = tables.Table()
        table.addHeader(("Owner UID", "Calendar Objects"))
        for uid, count in sorted(missing, key=lambda x: x[0]):
            table.addRow((
                uid,
                count,
            ))

        self.output.write("\n")
        self.logResult("Homes without a matching directory record", len(missing), uids_len)
        table.printTable(os=self.output)

        # Print table of results
        table = tables.Table()
        table.addHeader(("Owner UID", "Calendar Objects"))
        for uid, count in sorted(wrong_server, key=lambda x: x[0]):
            record = yield self.directoryService().recordWithUID(uid)
            table.addRow((
                "%s/%s (%s)" % (record.recordType if record else "-", record.shortNames[0] if record else "-", uid,),
                count,
            ))

        self.output.write("\n")
        self.logResult("Homes not hosted on this server", len(wrong_server), uids_len)
        table.printTable(os=self.output)

        # Print table of results
        table = tables.Table()
        table.addHeader(("Owner UID", "Calendar Objects"))
        for uid, count in sorted(disabled, key=lambda x: x[0]):
            record = yield self.directoryService().recordWithUID(uid)
            table.addRow((
                "%s/%s (%s)" % (record.recordType if record else "-", record.shortNames[0] if record else "-", uid,),
                count,
            ))

        self.output.write("\n")
        self.logResult("Homes without an enabled directory record", len(disabled), uids_len)
        table.printTable(os=self.output)

        self.printSummary()


class BadDataService(CalVerifyService):
    """
    Service which scans for bad calendar data.
    """

    def title(self):
        return "Bad Data Service"

    @inlineCallbacks
    def doAction(self):

        self.output.write("\n---- Scanning calendar data ----\n")

        self.now = DateTime.getNowUTC()
        self.start = DateTime.getToday()
        self.start.setDateOnly(False)
        self.end = self.start.duplicate()
        self.end.offsetYear(1)
        self.fix = self.options["fix"]

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")

        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()
        descriptor = None
        if self.options["uuid"]:
            rows = yield self.getAllResourceInfoWithUUID(self.options["uuid"], inbox=True, calendar=self.options["calendar"])
            descriptor = "getAllResourceInfoWithUUID"
        elif self.options["uid"]:
            rows = yield self.getAllResourceInfoWithUID(self.options["uid"], inbox=True)
            descriptor = "getAllResourceInfoWithUID"
        elif self.options["path"]:
            segments = self.options["path"].strip("/").split("/")
            if len(segments) not in (4, 3) or segments[0] != "calendars" or segments[1] != "__uids__":
                self.output.write("Invalid path for bad data scan\n")
                returnValue(None)
            else:
                uuid = segments[2]
                calendar = segments[3] if len(segments) == 4 else None
            rows = yield self.getAllResourceInfoWithUUID(uuid, inbox=True, calendar=calendar)
            descriptor = "getAllResourceInfoWithUUID"
        else:
            rows = yield self.getAllResourceInfo(inbox=True)
            descriptor = "getAllResourceInfo"

        yield self.txn.commit()
        self.txn = None

        if self.options["verbose"]:
            self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))

        self.total = len(rows)
        self.logResult("Number of events to process", self.total)
        self.addSummaryBreak()

        yield self.calendarDataCheck(rows)

        self.printSummary()

    @inlineCallbacks
    def calendarDataCheck(self, rows):
        """
        Check each calendar resource for valid iCalendar data.
        """

        self.output.write("\n---- Verifying each calendar object resource ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()

        results_bad = []
        count = 0
        total = len(rows)
        badlen = 0
        rjust = 10
        for owner, resid, uid, calname, _ignore_md5, _ignore_organizer, _ignore_created, _ignore_modified in rows:
            try:
                result, message = yield self.validCalendarData(resid, calname == "inbox")
            except Exception, e:
                result = False
                message = "Exception for validCalendarData"
                if self.options["verbose"]:
                    print(e)
            if not result:
                results_bad.append((owner, uid, resid, message))
                badlen += 1
            count += 1
            if self.options["verbose"]:
                if count == 1:
                    self.output.write("Bad".rjust(rjust) + "Current".rjust(rjust) + "Total".rjust(rjust) + "Complete".rjust(rjust) + "\n")
                    last_output = time.time()
                if divmod(count, 100)[1] == 0 or time.time() - last_output > 1:
                    self.output.write((
                        "\r" +
                        ("%s" % badlen).rjust(rjust) +
                        ("%s" % count).rjust(rjust) +
                        ("%s" % total).rjust(rjust) +
                        ("%d%%" % safePercent(count, total)).rjust(rjust) +
                        " " + calname + " " + str(resid)
                    ).ljust(80))
                    self.output.flush()
                    last_output = time.time()

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(count, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"]:
            self.output.write((
                "\r" +
                ("%s" % badlen).rjust(rjust) +
                ("%s" % count).rjust(rjust) +
                ("%s" % total).rjust(rjust) +
                ("%d%%" % safePercent(count, total)).rjust(rjust)
            ).ljust(80) + "\n")

        # Print table of results
        table = tables.Table()
        table.addHeader(("Owner", "Event UID", "RID", "Problem",))
        for item in sorted(results_bad, key=lambda x: (x[0], x[1])):
            owner, uid, resid, message = item
            owner_record = yield self.directoryService().recordWithUID(owner)
            table.addRow((
                "%s/%s (%s)" % (owner_record.recordType if owner_record else "-", owner_record.shortNames[0] if owner_record else "-", owner,),
                uid,
                resid,
                message,
            ))

        self.output.write("\n")
        self.logResult("Bad iCalendar data", len(results_bad), total)
        self.results["Bad iCalendar data"] = results_bad
        table.printTable(os=self.output)

        if self.options["verbose"]:
            diff_time = time.time() - t
            self.output.write("Time: %.2f s  Average: %.1f ms/resource\n" % (
                diff_time,
                safePercent(diff_time, total, 1000.0),
            ))

    errorPrefix = "Calendar data had unfixable problems:\n  "

    @inlineCallbacks
    def validCalendarData(self, resid, isinbox):
        """
        Check the calendar resource for valid iCalendar data.
        """

        result = True
        message = ""

        caldata = yield self.getCalendar(resid, self.fix)

        # Look for possible GEO fix
        if caldata is None and self.parseError and "GEO value incorrect" in self.parseException:
            if self.fix:
                caldata = yield self.fixInvalidGEOProperty(resid)
                if caldata is not None:
                    result = False
                    message = "Fixed: Removed bad GEO"
            else:
                self.parseError += ": GEO"

        if caldata is None:
            if self.parseError:
                returnValue((False, self.parseError))
            else:
                returnValue((True, "Nothing to scan"))

        component = Component(None, pycalendar=caldata)
        if getattr(self.config, "MaxInstancesForRRULE", 0):
            component.truncateRecurrence(self.config.MaxInstancesForRRULE)
        try:
            if self.options["ical"]:
                component.validCalendarData(doFix=False, validateRecurrences=True)
                component.validCalendarForCalDAV(methodAllowed=isinbox)
                component.validOrganizerForScheduling(doFix=False)
                if component.hasDuplicateAlarms(doFix=False):
                    raise InvalidICalendarDataError("Duplicate VALARMS")
            yield self.noPrincipalPathCUAddresses(component, doFix=False)
            if self.options["ical"]:
                self.attendeesWithoutOrganizer(component, doFix=False)

        except ValueError, e:
            result = False
            message = str(e)
            if message.startswith(self.errorPrefix):
                message = message[len(self.errorPrefix):]
            lines = message.splitlines()
            message = lines[0] + (" ++" if len(lines) > 1 else "")
            if self.fix:
                fixresult, fixmessage = yield self.fixCalendarData(resid, isinbox)
                if fixresult:
                    message = "Fixed: " + message
                else:
                    message = fixmessage + message

        returnValue((result, message,))

    @inlineCallbacks
    def noPrincipalPathCUAddresses(self, component, doFix):

        @inlineCallbacks
        def recordWithCalendarUserAddress(address):
            principal = yield self._principalCollection.principalForCalendarUserAddress(address)
            returnValue(principal.record)

        @inlineCallbacks
        def lookupFunction(cuaddr, recordFunction, conf):

            # Return cached results, if any.add
            if cuaddr in self.cuaCache:
                returnValue(self.cuaCache[cuaddr])

            result = yield normalizationLookup(cuaddr, recordFunction, conf)
            _ignore_name, guid, _ignore_cutype, _ignore_cuaddrs = result
            if guid is None:
                if cuaddr.find("__uids__") != -1:
                    guid = cuaddr[cuaddr.find("__uids__/") + 9:][:36]
                    result = ("", guid, "", set(),)

            # Cache the result
            self.cuaCache[cuaddr] = result
            returnValue(result)

        for subcomponent in component.subcomponents(ignore=True):
            organizer = subcomponent.getProperty("ORGANIZER")
            if organizer:
                cuaddr = organizer.value()

                # http(s) principals need to be converted to urn:uuid
                if cuaddr.startswith("http"):
                    if doFix:
                        yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)
                    else:
                        raise InvalidICalendarDataError("iCalendar ORGANIZER starts with 'http(s)'")
                elif cuaddr.startswith("mailto:"):
                    if (yield lookupFunction(cuaddr, recordWithCalendarUserAddress, self.config))[1] is not None:
                        if doFix:
                            yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)
                        else:
                            raise InvalidICalendarDataError("iCalendar ORGANIZER starts with 'mailto:' and record exists")
                else:
                    if ("@" in cuaddr) and (":" not in cuaddr) and ("/" not in cuaddr):
                        if doFix:
                            # Add back in mailto: then re-normalize to urn:uuid if possible
                            organizer.setValue("mailto:%s" % (cuaddr,))
                            yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)

                            # Remove any SCHEDULE-AGENT=NONE
                            if organizer.parameterValue("SCHEDULE-AGENT", "SERVER") == "NONE":
                                organizer.removeParameter("SCHEDULE-AGENT")
                        else:
                            raise InvalidICalendarDataError("iCalendar ORGANIZER missing mailto:")

            for attendee in subcomponent.properties("ATTENDEE"):
                cuaddr = attendee.value()

                # http(s) principals need to be converted to urn:uuid
                if cuaddr.startswith("http"):
                    if doFix:
                        yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)
                    else:
                        raise InvalidICalendarDataError("iCalendar ATTENDEE starts with 'http(s)'")
                elif cuaddr.startswith("mailto:"):
                    if (yield lookupFunction(cuaddr, recordWithCalendarUserAddress, self.config))[1] is not None:
                        if doFix:
                            yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)
                        else:
                            raise InvalidICalendarDataError("iCalendar ATTENDEE starts with 'mailto:' and record exists")
                else:
                    if ("@" in cuaddr) and (":" not in cuaddr) and ("/" not in cuaddr):
                        if doFix:
                            # Add back in mailto: then re-normalize to urn:uuid if possible
                            attendee.setValue("mailto:%s" % (cuaddr,))
                            yield component.normalizeCalendarUserAddresses(lookupFunction, recordWithCalendarUserAddress)
                        else:
                            raise InvalidICalendarDataError("iCalendar ATTENDEE missing mailto:")

    def attendeesWithoutOrganizer(self, component, doFix):
        """
        Look for events with ATTENDEE properties and no ORGANIZER property.
        """

        organizer = component.getOrganizer()
        attendees = component.getAttendees()
        if organizer is None and attendees:
            if doFix:
                raise ValueError("ATTENDEEs without ORGANIZER")
            else:
                raise InvalidICalendarDataError("ATTENDEEs without ORGANIZER")

    @inlineCallbacks
    def fixCalendarData(self, resid, isinbox):
        """
        Fix problems in calendar data using store APIs.
        """

        homeID, calendarID = yield self.getAllResourceInfoForResourceID(resid)
        home = yield self.txn.calendarHomeWithResourceID(homeID)
        calendar = yield home.childWithID(calendarID)
        calendarObj = yield calendar.objectResourceWithID(resid)

        try:
            component = yield calendarObj.component()
        except InternalDataStoreError:
            returnValue((False, "Failed parse: "))

        result = True
        message = ""
        try:
            if self.options["ical"]:
                component.validCalendarData(doFix=True, validateRecurrences=True)
                component.validCalendarForCalDAV(methodAllowed=isinbox)
                component.validOrganizerForScheduling(doFix=True)
                component.hasDuplicateAlarms(doFix=True)
            yield self.noPrincipalPathCUAddresses(component, doFix=True)
            if self.options["ical"]:
                self.attendeesWithoutOrganizer(component, doFix=True)
        except ValueError:
            result = False
            message = "Failed fix: "

        if result:
            # Write out fix, commit and get a new transaction
            try:
                # Use _migrating to ignore possible overridden instance errors - we are either correcting or ignoring those
                self.txn._migrating = True
                component = yield calendarObj._setComponentInternal(component, internal_state=ComponentUpdateState.RAW)
            except Exception, e:
                print(e, component)
                print(traceback.print_exc())
                result = False
                message = "Exception fix: "
            yield self.txn.commit()
            self.txn = self.store.newTransaction()

        returnValue((result, message,))

    @inlineCallbacks
    def fixInvalidGEOProperty(self, resid, doFix=False):
        self.parseError = None
        self.parseException = None
        co = schema.CALENDAR_OBJECT
        kwds = {"ResourceID": resid}
        rows = (yield Select(
            [co.ICALENDAR_TEXT],
            From=co,
            Where=(
                co.RESOURCE_ID == Parameter("ResourceID")
            ),
        ).on(self.txn, **kwds))
        caltextdata = rows[0][0] if rows else None
        if caltextdata is None:
            returnValue(None)
        caltextdata = "\n".join(filter(lambda x: not (x.startswith("GEO:") and ";" not in x), caltextdata.splitlines()))

        try:
            caldata = Calendar.parseText(caltextdata)
        except ErrorBase as e:
            self.parseError = "Failed to parse"
            self.parseException = str(e)
            returnValue(None)

        # We now know it is valid, so write it back
        yield Update(
            {co.ICALENDAR_TEXT: caltextdata},
            Where=co.RESOURCE_ID == Parameter("ResourceID")
        ).on(self.txn, **kwds)

        returnValue(caldata)


class SchedulingMismatchService(CalVerifyService):
    """
    Service which detects mismatched scheduled events.
    """

    metadata = {
        "accessMode": "PUBLIC",
        "isScheduleObject": True,
        "scheduleTag": "abc",
        "scheduleEtags": (),
        "hasPrivateComment": False,
    }

    metadata_inbox = {
        "accessMode": "PUBLIC",
        "isScheduleObject": False,
        "scheduleTag": "",
        "scheduleEtags": (),
        "hasPrivateComment": False,
    }

    def __init__(self, store, options, output, reactor, config):
        super(SchedulingMismatchService, self).__init__(store, options, output, reactor, config)

        self.validForCalendaringUUIDs = {}

        self.fixAttendeesForOrganizerMissing = 0
        self.fixAttendeesForOrganizerMismatch = 0
        self.fixOrganizersForAttendeeMissing = 0
        self.fixOrganizersForAttendeeMismatch = 0
        self.fixFailed = 0
        self.fixedAutoAccepts = []

    def title(self):
        return "Scheduling Mismatch Service"

    @inlineCallbacks
    def doAction(self):

        self.output.write("\n---- Scanning calendar data ----\n")

        self.now = DateTime.getNowUTC()
        self.start = self.options["start"] if "start" in self.options else DateTime.getToday()
        self.start.setDateOnly(False)
        self.end = self.start.duplicate()
        self.end.offsetYear(1)
        self.fix = self.options["fix"]

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")

        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()
        descriptor = None
        if self.options["uid"]:
            rows = yield self.getAllResourceInfoWithUID(self.options["uid"])
            descriptor = "getAllResourceInfoWithUID"
        elif self.options["uuid"]:
            rows = yield self.getAllResourceInfoTimeRangeWithUUIDForAllUID(self.start, self.options["uuid"])
            descriptor = "getAllResourceInfoTimeRangeWithUUIDForAllUID"
            self.options["uuid"] = None
        else:
            rows = yield self.getAllResourceInfoTimeRange(self.start)
            descriptor = "getAllResourceInfoTimeRange"

        yield self.txn.commit()
        self.txn = None

        if self.options["verbose"]:
            self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))

        self.total = len(rows)
        self.logResult("Number of events to process", self.total)

        # Split into organizer events and attendee events
        self.organized = []
        self.organized_byuid = {}
        self.attended = []
        self.attended_byuid = collections.defaultdict(list)
        self.matched_attendee_to_organizer = collections.defaultdict(set)
        skipped, inboxes = yield self.buildResourceInfo(rows)

        self.logResult("Number of organizer events to process", len(self.organized), self.total)
        self.logResult("Number of attendee events to process", len(self.attended), self.total)
        self.logResult("Number of skipped events", skipped, self.total)
        self.logResult("Number of inbox events", inboxes)
        self.addSummaryBreak()

        self.totalErrors = 0
        yield self.verifyAllAttendeesForOrganizer()
        yield self.verifyAllOrganizersForAttendee()

        # Need to add fix summary information
        if self.fix:
            self.addSummaryBreak()
            self.logResult("Fixed missing attendee events", self.fixAttendeesForOrganizerMissing)
            self.logResult("Fixed mismatched attendee events", self.fixAttendeesForOrganizerMismatch)
            self.logResult("Fixed missing organizer events", self.fixOrganizersForAttendeeMissing)
            self.logResult("Fixed mismatched organizer events", self.fixOrganizersForAttendeeMismatch)
            self.logResult("Fix failures", self.fixFailed)
            self.logResult("Fixed Auto-Accepts", len(self.fixedAutoAccepts))
            self.results["Auto-Accepts"] = self.fixedAutoAccepts

            self.printAutoAccepts()

        self.printSummary()

    @inlineCallbacks
    def buildResourceInfo(self, rows, onlyOrganizer=False, onlyAttendee=False):
        """
        For each resource, determine whether it is an organizer or attendee event, and also
        cache the attendee partstats.

        @param rows: set of DB query rows
        @type rows: C{list}
        @param onlyOrganizer: whether organizer information only is required
        @type onlyOrganizer: C{bool}
        @param onlyAttendee: whether attendee information only is required
        @type onlyAttendee: C{bool}
        """

        skipped = 0
        inboxes = 0
        for owner, resid, uid, calname, md5, organizer, created, modified in rows:

            # Skip owners not enabled for calendaring
            if not (yield self.testForCalendaringUUID(owner)):
                skipped += 1
                continue

            # Skip inboxes
            if calname == "inbox":
                inboxes += 1
                continue

            # If targeting a specific organizer, skip events belonging to others
            if self.options["uuid"]:
                if not organizer.startswith("urn:x-uid:") or self.options["uuid"] != organizer[10:]:
                    continue

            # Cache organizer/attendee states
            if organizer.startswith("urn:x-uid:") and owner == organizer[10:]:
                if not onlyAttendee:
                    self.organized.append((owner, resid, uid, md5, organizer, created, modified,))
                    self.organized_byuid[uid] = (owner, resid, uid, md5, organizer, created, modified,)
            else:
                if not onlyOrganizer:
                    self.attended.append((owner, resid, uid, md5, organizer, created, modified,))
                    self.attended_byuid[uid].append((owner, resid, uid, md5, organizer, created, modified,))

        returnValue((skipped, inboxes))

    @inlineCallbacks
    def testForCalendaringUUID(self, uuid):
        """
        Determine if the specified directory UUID is valid for calendaring. Keep a cache of
        valid and invalid so we can do this quickly.

        @param uuid: the directory UUID to test
        @type uuid: C{str}

        @return: C{True} if valid, C{False} if not
        """

        if uuid not in self.validForCalendaringUUIDs:
            record = yield self.directoryService().recordWithUID(uuid)
            self.validForCalendaringUUIDs[uuid] = record is not None and record.hasCalendars and record.thisServer()
        returnValue(self.validForCalendaringUUIDs[uuid])

    @inlineCallbacks
    def verifyAllAttendeesForOrganizer(self):
        """
        Make sure that for each organizer, each referenced attendee has a consistent view of the organizer's event.
        We will look for events that an organizer has and are missing for the attendee, and events that an organizer's
        view of attendee status does not match the attendee's view of their own status.
        """

        self.output.write("\n---- Verifying Organizer events against Attendee copies ----\n")
        self.txn = self.store.newTransaction()

        results_missing = []
        results_mismatch = []
        attendeeResIDs = {}
        organized_len = len(self.organized)
        organizer_div = 1 if organized_len < 100 else organized_len / 100

        # Test organized events
        t = time.time()
        for ctr, organizerEvent in enumerate(self.organized):
            if self.options["verbose"] and divmod(ctr, organizer_div)[1] == 0:
                self.output.write(("\r%d of %d (%d%%) Missing: %d  Mismatched: %s" % (
                    ctr + 1,
                    organized_len,
                    ((ctr + 1) * 100 / organized_len),
                    len(results_missing),
                    len(results_mismatch),
                )).ljust(80))
                self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 10 seconds
            if time.time() - t > 10:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()
                t = time.time()

            # Get the organizer's view of attendee states
            organizer, resid, uid, _ignore_md5, _ignore_organizer, org_created, org_modified = organizerEvent
            calendar = yield self.getCalendar(resid)
            if calendar is None:
                continue
            if self.options["verbose"] and self.masterComponent(calendar) is None:
                self.output.write("Missing master for organizer: %s, resid: %s, uid: %s\n" % (organizer, resid, uid,))
            organizerViewOfAttendees = self.buildAttendeeStates(calendar, self.start, self.end)
            try:
                del organizerViewOfAttendees[organizer]
            except KeyError:
                # Odd - the organizer is not an attendee - this usually does not happen
                pass
            if len(organizerViewOfAttendees) == 0:
                continue

            # Get attendee states for matching UID
            eachAttendeesOwnStatus = {}
            attendeeCreatedModified = {}
            for attendeeEvent in self.attended_byuid.get(uid, ()):
                owner, attresid, attuid, _ignore_md5, _ignore_organizer, att_created, att_modified = attendeeEvent
                attendeeCreatedModified[owner] = (att_created, att_modified,)
                calendar = yield self.getCalendar(attresid)
                if calendar is None:
                    continue
                eachAttendeesOwnStatus[owner] = self.buildAttendeeStates(calendar, self.start, self.end, attendee_only=owner)
                attendeeResIDs[(owner, attuid)] = attresid

            # Look at each attendee in the organizer's meeting
            for organizerAttendee, organizerViewOfStatus in organizerViewOfAttendees.iteritems():
                missing = False
                mismatch = False

                self.matched_attendee_to_organizer[uid].add(organizerAttendee)

                # Skip attendees not enabled for calendaring
                if not (yield self.testForCalendaringUUID(organizerAttendee)):
                    continue

                # Double check the missing attendee situation in case we missed it during the original query
                if organizerAttendee not in eachAttendeesOwnStatus:
                    # Try to reload the attendee data
                    calendar, attresid, att_created, att_modified = yield self.getCalendarForOwnerByUID(organizerAttendee, uid)
                    if calendar is not None:
                        eachAttendeesOwnStatus[organizerAttendee] = self.buildAttendeeStates(calendar, self.start, self.end, attendee_only=organizerAttendee)
                        attendeeResIDs[(organizerAttendee, uid)] = attresid
                        attendeeCreatedModified[organizerAttendee] = (att_created, att_modified,)
                        # print("Reloaded missing attendee data")

                # If an entry for the attendee exists, then check whether attendee status matches
                if organizerAttendee in eachAttendeesOwnStatus:
                    attendeeOwnStatus = eachAttendeesOwnStatus[organizerAttendee].get(organizerAttendee, set())
                    att_created, att_modified = attendeeCreatedModified[organizerAttendee]

                    if organizerViewOfStatus != attendeeOwnStatus:
                        # Check that the difference is only cancelled or declined on the organizers side
                        for _organizerInstance, partstat in organizerViewOfStatus.difference(attendeeOwnStatus):
                            if partstat not in ("DECLINED", "CANCELLED"):
                                results_mismatch.append((uid, resid, organizer, org_created, org_modified, organizerAttendee, att_created, att_modified))
                                self.results.setdefault("Mismatch Attendee", set()).add((uid, organizer, organizerAttendee,))
                                mismatch = True
                                if self.options["details"]:
                                    self.output.write("Mismatch: on Organizer's side:\n")
                                    self.output.write("          UID: %s\n" % (uid,))
                                    self.output.write("          Organizer: %s\n" % (organizer,))
                                    self.output.write("          Attendee: %s\n" % (organizerAttendee,))
                                    self.output.write("          Instance: %s\n" % (_organizerInstance,))
                                break
                        # Check that the difference is only cancelled on the attendees side
                        for _attendeeInstance, partstat in attendeeOwnStatus.difference(organizerViewOfStatus):
                            if partstat not in ("CANCELLED",):
                                if not mismatch:
                                    results_mismatch.append((uid, resid, organizer, org_created, org_modified, organizerAttendee, att_created, att_modified))
                                    self.results.setdefault("Mismatch Attendee", set()).add((uid, organizer, organizerAttendee,))
                                mismatch = True
                                if self.options["details"]:
                                    self.output.write("Mismatch: on Attendee's side:\n")
                                    self.output.write("          Organizer: %s\n" % (organizer,))
                                    self.output.write("          Attendee: %s\n" % (organizerAttendee,))
                                    self.output.write("          Instance: %s\n" % (_attendeeInstance,))
                                break

                # Check that the status for this attendee is always declined which means a missing copy of the event is OK
                else:
                    for _ignore_instance_id, partstat in organizerViewOfStatus:
                        if partstat not in ("DECLINED", "CANCELLED"):
                            results_missing.append((uid, resid, organizer, organizerAttendee, org_created, org_modified))
                            self.results.setdefault("Missing Attendee", set()).add((uid, organizer, organizerAttendee,))
                            missing = True
                            break

                # If there was a problem we can fix it
                if (missing or mismatch) and self.fix:
                    fix_result = (yield self.fixByReinvitingAttendee(resid, attendeeResIDs.get((organizerAttendee, uid)), organizerAttendee))
                    if fix_result:
                        if missing:
                            self.fixAttendeesForOrganizerMissing += 1
                        else:
                            self.fixAttendeesForOrganizerMismatch += 1
                    else:
                        self.fixFailed += 1

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"]:
            self.output.write("\r".ljust(80) + "\n")

        # Print table of results
        table = tables.Table()
        table.addHeader(("Organizer", "Attendee", "Event UID", "Organizer RID", "Created", "Modified",))
        results_missing.sort()
        for item in results_missing:
            uid, resid, organizer, attendee, created, modified = item
            organizer_record = yield self.directoryService().recordWithUID(organizer)
            attendee_record = yield self.directoryService().recordWithUID(attendee)
            table.addRow((
                "%s/%s (%s)" % (organizer_record.recordType if organizer_record else "-", organizer_record.shortNames[0] if organizer_record else "-", organizer,),
                "%s/%s (%s)" % (attendee_record.recordType if attendee_record else "-", attendee_record.shortNames[0] if attendee_record else "-", attendee,),
                uid,
                resid,
                created,
                "" if modified == created else modified,
            ))

        self.output.write("\n")
        self.logResult("Events missing from Attendee's calendars", len(results_missing), self.total)
        table.printTable(os=self.output)
        self.totalErrors += len(results_missing)

        # Print table of results
        table = tables.Table()
        table.addHeader(("Organizer", "Attendee", "Event UID", "Organizer RID", "Created", "Modified", "Attendee RID", "Created", "Modified",))
        results_mismatch.sort()
        for item in results_mismatch:
            uid, org_resid, organizer, org_created, org_modified, attendee, att_created, att_modified = item
            organizer_record = yield self.directoryService().recordWithUID(organizer)
            attendee_record = yield self.directoryService().recordWithUID(attendee)
            table.addRow((
                "%s/%s (%s)" % (organizer_record.recordType if organizer_record else "-", organizer_record.shortNames[0] if organizer_record else "-", organizer,),
                "%s/%s (%s)" % (attendee_record.recordType if attendee_record else "-", attendee_record.shortNames[0] if attendee_record else "-", attendee,),
                uid,
                org_resid,
                org_created,
                "" if org_modified == org_created else org_modified,
                attendeeResIDs[(attendee, uid)],
                att_created,
                "" if att_modified == att_created else att_modified,
            ))

        self.output.write("\n")
        self.logResult("Events mismatched between Organizer's and Attendee's calendars", len(results_mismatch), self.total)
        table.printTable(os=self.output)
        self.totalErrors += len(results_mismatch)

    @inlineCallbacks
    def verifyAllOrganizersForAttendee(self):
        """
        Make sure that for each attendee, there is a matching event for the organizer.
        """

        self.output.write("\n---- Verifying Attendee events against Organizer copies ----\n")
        self.txn = self.store.newTransaction()

        # Now try to match up each attendee event
        missing = []
        mismatched = []
        attended_len = len(self.attended)
        attended_div = 1 if attended_len < 100 else attended_len / 100

        t = time.time()
        for ctr, attendeeEvent in enumerate(tuple(self.attended)):  # self.attended might mutate during the loop

            if self.options["verbose"] and divmod(ctr, attended_div)[1] == 0:
                self.output.write(("\r%d of %d (%d%%) Missing: %d  Mismatched: %s" % (
                    ctr + 1,
                    attended_len,
                    ((ctr + 1) * 100 / attended_len),
                    len(missing),
                    len(mismatched),
                )).ljust(80))
                self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 10 seconds
            if time.time() - t > 10:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()
                t = time.time()

            attendee, resid, uid, _ignore_md5, organizer, att_created, att_modified = attendeeEvent
            calendar = yield self.getCalendar(resid)
            if calendar is None:
                continue
            eachAttendeesOwnStatus = self.buildAttendeeStates(calendar, self.start, self.end, attendee_only=attendee)
            if attendee not in eachAttendeesOwnStatus:
                continue

            # Only care about data for hosted organizers
            if not organizer.startswith("urn:x-uid:"):
                continue
            organizer = organizer[10:]

            # Skip organizers not enabled for calendaring
            if not (yield self.testForCalendaringUUID(organizer)):
                continue

            # Double check the missing attendee situation in case we missed it during the original query
            if uid not in self.organized_byuid:
                # Try to reload the organizer info data
                rows = yield self.getAllResourceInfoWithUID(uid)
                yield self.buildResourceInfo(rows, onlyOrganizer=True)

                # if uid in self.organized_byuid:
                #     print("Reloaded missing organizer data: %s" % (uid,))

            if uid not in self.organized_byuid:

                # Check whether attendee has all instances cancelled
                if self.allCancelled(eachAttendeesOwnStatus):
                    continue

                missing.append((uid, attendee, organizer, resid, att_created, att_modified,))
                self.results.setdefault("Missing Organizer", set()).add((uid, attendee, organizer,))

                # If there is a miss we fix by removing the attendee data
                if self.fix:
                    # This is where we attempt a fix
                    fix_result = (yield self.removeEvent(resid))
                    if fix_result:
                        self.fixOrganizersForAttendeeMissing += 1
                    else:
                        self.fixFailed += 1

            elif attendee not in self.matched_attendee_to_organizer[uid]:
                # Check whether attendee has all instances cancelled
                if self.allCancelled(eachAttendeesOwnStatus):
                    continue

                mismatched.append((uid, attendee, organizer, resid, att_created, att_modified,))
                self.results.setdefault("Mismatch Organizer", set()).add((uid, attendee, organizer,))

                # If there is a mismatch we fix by re-inviting the attendee
                if self.fix:
                    fix_result = (yield self.fixByReinvitingAttendee(self.organized_byuid[uid][1], resid, attendee))
                    if fix_result:
                        self.fixOrganizersForAttendeeMismatch += 1
                    else:
                        self.fixFailed += 1

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"]:
            self.output.write("\r".ljust(80) + "\n")

        # Print table of results
        table = tables.Table()
        table.addHeader(("Organizer", "Attendee", "UID", "Attendee RID", "Created", "Modified",))
        missing.sort()
        unique_set = set()
        for item in missing:
            uid, attendee, organizer, resid, created, modified = item
            unique_set.add(uid)
            if organizer:
                organizerRecord = yield self.directoryService().recordWithUID(organizer)
                organizer = "%s/%s (%s)" % (organizerRecord.recordType if organizerRecord else "-", organizerRecord.shortNames[0] if organizerRecord else "-", organizer,)
            attendeeRecord = yield self.directoryService().recordWithUID(attendee)
            table.addRow((
                organizer,
                "%s/%s (%s)" % (attendeeRecord.recordType if attendeeRecord else "-", attendeeRecord.shortNames[0] if attendeeRecord else "-", attendee,),
                uid,
                resid,
                created,
                "" if modified == created else modified,
            ))

        self.output.write("\n")
        self.output.write("Attendee events missing in Organizer's calendar (total=%d, unique=%d):\n" % (len(missing), len(unique_set),))
        table.printTable(os=self.output)
        self.addToSummary("Attendee events missing in Organizer's calendar", len(missing), self.total)
        self.totalErrors += len(missing)

        # Print table of results
        table = tables.Table()
        table.addHeader(("Organizer", "Attendee", "UID", "Organizer RID", "Created", "Modified", "Attendee RID", "Created", "Modified",))
        mismatched.sort()
        for item in mismatched:
            uid, attendee, organizer, resid, att_created, att_modified = item
            if organizer:
                organizerRecord = yield self.directoryService().recordWithUID(organizer)
                organizer = "%s/%s (%s)" % (organizerRecord.recordType if organizerRecord else "-", organizerRecord.shortNames[0] if organizerRecord else "-", organizer,)
            attendeeRecord = yield self.directoryService().recordWithUID(attendee)
            table.addRow((
                organizer,
                "%s/%s (%s)" % (attendeeRecord.recordType if attendeeRecord else "-", attendeeRecord.shortNames[0] if attendeeRecord else "-", attendee,),
                uid,
                self.organized_byuid[uid][1],
                self.organized_byuid[uid][5],
                self.organized_byuid[uid][6],
                resid,
                att_created,
                "" if att_modified == att_created else att_modified,
            ))

        self.output.write("\n")
        self.logResult("Attendee events mismatched in Organizer's calendar", len(mismatched), self.total)
        table.printTable(os=self.output)
        self.totalErrors += len(mismatched)

    @inlineCallbacks
    def fixByReinvitingAttendee(self, orgresid, attresid, attendee):
        """
        Fix a mismatch/missing error by having the organizer send a REQUEST for the entire event to the attendee
        to trigger implicit scheduling to resync the attendee event.

        We do not have implicit apis in the store, but really want to use store-only apis here to avoid having to create
        "fake" HTTP requests and manipulate HTTP resources. So what we will do is emulate implicit behavior by copying the
        organizer resource to the attendee (filtering it for the attendee's view of the event) and deposit an inbox item
        for the same event. Right now that will wipe out any per-attendee data - notably alarms.
        """

        try:
            cuaddr = "urn:x-uid:%s" % attendee

            # Get the organizer's calendar data
            calendar = (yield self.getCalendar(orgresid))
            calendar = Component(None, pycalendar=calendar)

            # Generate an iTip message for the entire event filtered for the attendee's view
            itipmsg = iTipGenerator.generateAttendeeRequest(calendar, (cuaddr,), None)

            # Handle the case where the attendee is not actually in the organizer event at all by
            # removing the attendee event instead of re-inviting
            if itipmsg.resourceUID() is None:
                yield self.removeEvent(attresid)
                returnValue(True)

            # Convert iTip message into actual calendar data - just remove METHOD
            attendee_calendar = itipmsg.duplicate()
            attendee_calendar.removeProperty(attendee_calendar.getProperty("METHOD"))

            # Adjust TRANSP to match PARTSTAT
            self.setTransparencyForAttendee(attendee_calendar, cuaddr)

            # Get attendee home store object
            home = (yield self.txn.calendarHomeWithUID(attendee))
            if home is None:
                raise ValueError("Cannot find home")
            inbox = (yield home.calendarWithName("inbox"))
            if inbox is None:
                raise ValueError("Cannot find inbox")

            details = {}
            # Replace existing resource data, or create a new one
            if attresid:
                # TODO: transfer over per-attendee data - valarms
                _ignore_homeID, calendarID = yield self.getAllResourceInfoForResourceID(attresid)
                calendar = yield home.childWithID(calendarID)
                calendarObj = yield calendar.objectResourceWithID(attresid)
                calendarObj.scheduleTag = str(uuid4())
                yield calendarObj._setComponentInternal(attendee_calendar, internal_state=ComponentUpdateState.RAW)
                self.results.setdefault("Fix change event", set()).add((home.name(), calendar.name(), attendee_calendar.resourceUID(),))

                details["path"] = "/calendars/__uids__/%s/%s/%s" % (home.name(), calendar.name(), calendarObj.name(),)
                details["rid"] = attresid
            else:
                # Find default calendar for VEVENTs
                defaultCalendar = (yield self.defaultCalendarForAttendee(home))
                if defaultCalendar is None:
                    raise ValueError("Cannot find suitable default calendar")
                new_name = str(uuid4()) + ".ics"
                calendarObj = (yield defaultCalendar._createCalendarObjectWithNameInternal(new_name, attendee_calendar, internal_state=ComponentUpdateState.RAW, options=self.metadata))
                self.results.setdefault("Fix add event", set()).add((home.name(), defaultCalendar.name(), attendee_calendar.resourceUID(),))

                details["path"] = "/calendars/__uids__/%s/%s/%s" % (home.name(), defaultCalendar.name(), new_name,)
                details["rid"] = calendarObj._resourceID

            details["uid"] = attendee_calendar.resourceUID()
            instances = attendee_calendar.expandTimeRanges(self.end)
            for key in instances:
                instance = instances[key]
                if instance.start > self.now:
                    break
            details["start"] = instance.start.adjustTimezone(self.tzid)
            details["title"] = instance.component.propertyValue("SUMMARY")

            # Write new itip message to attendee inbox
            yield inbox.createCalendarObjectWithName(str(uuid4()) + ".ics", itipmsg, options=self.metadata_inbox)
            self.results.setdefault("Fix add inbox", set()).add((home.name(), itipmsg.resourceUID(),))

            yield self.txn.commit()
            self.txn = self.store.newTransaction()

            # Need to know whether the attendee is a location or resource with auto-accept set
            record = yield self.directoryService().recordWithUID(attendee)
            autoScheduleMode = getattr(record, "autoScheduleMode", None)
            if autoScheduleMode not in (None, AutoScheduleMode.none):
                # Log details about the event so we can have a human manually process
                self.fixedAutoAccepts.append(details)

            returnValue(True)

        except Exception, e:
            print("Failed to fix resource: %d for attendee: %s\n%s" % (orgresid, attendee, e,))
            returnValue(False)

    @inlineCallbacks
    def defaultCalendarForAttendee(self, home):

        # Check for property
        calendar = (yield home.defaultCalendar("VEVENT"))
        returnValue(calendar)

    def printAutoAccepts(self):
        # Print summary of results
        table = tables.Table()
        table.addHeader(("Path", "RID", "UID", "Start Time", "Title"))
        for item in sorted(self.fixedAutoAccepts, key=lambda x: x["path"]):
            table.addRow((
                item["path"],
                item["rid"],
                item["uid"],
                item["start"],
                item["title"],
            ))

        self.output.write("\n")
        self.output.write("Auto-Accept Fixes:\n")
        table.printTable(os=self.output)

    def masterComponent(self, calendar):
        """
        Return the master iCal component in this calendar.

        @return: the L{Component} for the master component,
            or C{None} if there isn't one.
        """
        for component in calendar.getComponents(definitions.cICalComponent_VEVENT):
            if not component.hasProperty("RECURRENCE-ID"):
                return component

        return None

    def buildAttendeeStates(self, calendar, start, end, attendee_only=None):
        # Expand events into instances in the start/end range
        results = []
        calendar.getVEvents(
            Period(
                start=start,
                end=end,
            ),
            results
        )

        # Need to do iCal fake master fixup
        overrides = len(calendar.getComponents(definitions.cICalComponent_VEVENT)) > 1

        # Create map of each attendee's instances with the instance id (start time) and attendee part-stat
        attendees = {}
        for item in results:

            # Fake master fixup
            if overrides:
                if not item.getOwner().isRecurrenceInstance():
                    if item.getOwner().getRecurrenceSet() is None or not item.getOwner().getRecurrenceSet().hasRecurrence():
                        continue

            # Get Status - ignore cancelled events
            status = item.getOwner().loadValueString(definitions.cICalProperty_STATUS)
            cancelled = status == definitions.cICalProperty_STATUS_CANCELLED

            # Get instance start
            item.getInstanceStart().adjustToUTC()
            instance_id = item.getInstanceStart().getText()

            props = item.getOwner().getProperties().get(definitions.cICalProperty_ATTENDEE, [])
            for prop in props:
                caladdr = prop.getCalAddressValue().getValue()
                if caladdr.startswith("urn:x-uid:"):
                    caladdr = caladdr[10:]
                else:
                    continue
                if attendee_only is not None and attendee_only != caladdr:
                    continue
                if cancelled:
                    partstat = "CANCELLED"
                else:
                    if not prop.hasParameter(definitions.cICalParameter_PARTSTAT):
                        partstat = definitions.cICalParameter_PARTSTAT_NEEDSACTION
                    else:
                        partstat = prop.getParameterValue(definitions.cICalParameter_PARTSTAT)

                attendees.setdefault(caladdr, set()).add((instance_id, partstat))

        return attendees

    def allCancelled(self, attendeesStatus):
        # Check whether attendees have all instances cancelled
        all_cancelled = True
        for _ignore_guid, states in attendeesStatus.iteritems():
            for _ignore_instance_id, partstat in states:
                if partstat not in ("CANCELLED", "DECLINED",):
                    all_cancelled = False
                    break
            if not all_cancelled:
                break
        return all_cancelled

    def setTransparencyForAttendee(self, calendar, attendee):
        """
        Set the TRANSP property based on the PARTSTAT value on matching ATTENDEE properties
        in each component.
        """
        for component in calendar.subcomponents(ignore=True):
            prop = component.getAttendeeProperty(attendee)
            addTransp = False
            if prop:
                partstat = prop.parameterValue("PARTSTAT", "NEEDS-ACTION")
                addTransp = partstat in ("NEEDS-ACTION", "DECLINED",)
            component.replaceProperty(Property("TRANSP", "TRANSPARENT" if addTransp else "OPAQUE"))


class DoubleBookingService(CalVerifyService):
    """
    Service which detects double-booked events.
    """

    def title(self):
        return "Double Booking Service"

    @inlineCallbacks
    def doAction(self):

        if self.options["fix"]:
            self.output.write("\nFixing is not supported.\n")
            returnValue(None)

        self.output.write("\n---- Scanning calendar data ----\n")

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")
        self.now = DateTime.getNowUTC()
        self.start = DateTime.getToday()
        self.start.setDateOnly(False)
        self.start.setTimezone(self.tzid)
        self.end = self.start.duplicate()
        self.end.offsetYear(1)
        self.fix = self.options["fix"]

        if self.options["verbose"] and self.options["summary"]:
            ot = time.time()

        # Check loop over uuid
        UUIDDetails = collections.namedtuple("UUIDDetails", ("uuid", "rname", "auto", "doubled",))
        self.uuid_details = []
        if len(self.options["uuid"]) != 36:
            self.txn = self.store.newTransaction()
            if self.options["uuid"]:
                homes = yield self.getMatchingHomeUIDs(self.options["uuid"])
            else:
                homes = yield self.getAllHomeUIDs()
            yield self.txn.commit()
            self.txn = None
            uuids = []
            for uuid in sorted(homes):
                record = yield self.directoryService().recordWithUID(uuid)
                if record is not None and record.recordType in (CalRecordType.location, CalRecordType.resource):
                    uuids.append(uuid)
        else:
            uuids = [self.options["uuid"], ]

        count = 0
        for uuid in uuids:
            self.results = {}
            self.summary = []
            self.total = 0
            count += 1

            record = yield self.directoryService().recordWithUID(uuid)
            if record is None:
                continue
            if not record.thisServer() or not record.hasCalendars:
                continue

            rname = record.displayName
            autoScheduleMode = getattr(record, "autoSchedule", AutoScheduleMode.none)

            if len(uuids) > 1 and not self.options["summary"]:
                self.output.write("\n\n-----------------------------\n")

            self.txn = self.store.newTransaction()

            if self.options["verbose"]:
                t = time.time()
            rows = yield self.getTimeRangeInfoWithUUID(uuid, self.start)
            descriptor = "getTimeRangeInfoWithUUID"

            yield self.txn.commit()
            self.txn = None

            if self.options["verbose"]:
                if not self.options["summary"]:
                    self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))
                else:
                    self.output.write("%s (%d/%d)" % (uuid, count, len(uuids),))
                    self.output.flush()

            self.total = len(rows)
            if not self.options["summary"]:
                self.logResult("UUID to process", uuid)
                self.logResult("Record name", rname)
                self.logResult("Auto-schedule-mode", autoScheduleMode.description)
                self.addSummaryBreak()
                self.logResult("Number of events to process", self.total)

            if rows:
                if not self.options["summary"]:
                    self.addSummaryBreak()
                doubled = yield self.doubleBookCheck(rows, uuid, self.start)
            else:
                doubled = False

            self.uuid_details.append(UUIDDetails(uuid, rname, autoScheduleMode, doubled))

            if not self.options["summary"]:
                self.printSummary()
            else:
                self.output.write(" - %s\n" % ("Double-booked" if doubled else "OK",))
                self.output.flush()

        if self.options["summary"]:
            table = tables.Table()
            table.addHeader(("GUID", "Name", "Auto-Schedule", "Double-Booked",))
            doubled = 0
            for item in sorted(self.uuid_details):
                if not item.doubled:
                    continue
                table.addRow((
                    item.uuid,
                    item.rname,
                    item.autoScheduleMode,
                    item.doubled,
                ))
                doubled += 1
            table.addFooter(("Total", "", "", "%d of %d" % (doubled, len(self.uuid_details),),))
            self.output.write("\n")
            table.printTable(os=self.output)

            if self.options["verbose"]:
                self.output.write("%s time: %.1fs\n" % ("Summary", time.time() - ot,))

    @inlineCallbacks
    def getTimeRangeInfoWithUUID(self, uuid, start):
        co = schema.CALENDAR_OBJECT
        cb = schema.CALENDAR_BIND
        ch = schema.CALENDAR_HOME
        tr = schema.TIME_RANGE
        kwds = {
            "uuid": uuid,
            "Start": pyCalendarToSQLTimestamp(start),
        }
        rows = (yield Select(
            [co.RESOURCE_ID, ],
            From=ch.join(
                cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
                co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
                    cb.BIND_MODE == _BIND_MODE_OWN).And(
                    cb.CALENDAR_RESOURCE_NAME != "inbox").And(
                    co.ORGANIZER != "")).join(
                tr, type="left", on=(co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID)),
            Where=(ch.OWNER_UID == Parameter("uuid")).And((tr.START_DATE >= Parameter("Start")).Or(co.RECURRANCE_MAX <= Parameter("Start"))),
            Distinct=True,
        ).on(self.txn, **kwds))
        returnValue(tuple(rows))

    @inlineCallbacks
    def doubleBookCheck(self, rows, uuid, start):
        """
        Check each calendar resource by expanding instances within the next year, and looking for
        any that overlap with status not CANCELLED and PARTSTAT ACCEPTED.
        """

        if not self.options["summary"]:
            self.output.write("\n---- Checking instances for double-booking ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()

        InstanceDetails = collections.namedtuple("InstanceDetails", ("resid", "uid", "start", "end", "organizer", "summary",))

        end = start.duplicate()
        end.offsetDay(int(self.options["days"]))
        count = 0
        total = len(rows)
        total_instances = 0
        booked_instances = 0
        details = []
        rjust = 10
        tzid = None
        hasFloating = False
        for resid in rows:
            resid = resid[0]
            caldata = yield self.getCalendar(resid, self.fix)
            if caldata is None:
                if self.parseError:
                    returnValue((False, self.parseError))
                else:
                    returnValue((True, "Nothing to scan"))

            cal = Component(None, pycalendar=caldata)
            cal = PerUserDataFilter(uuid).filter(cal)
            uid = cal.resourceUID()
            instances = cal.expandTimeRanges(end, start, ignoreInvalidInstances=False)
            count += 1

            for instance in instances.instances.values():
                total_instances += 1

                # See if it is CANCELLED or TRANSPARENT
                if instance.component.propertyValue("STATUS") == "CANCELLED":
                    continue
                if instance.component.propertyValue("TRANSP") == "TRANSPARENT":
                    continue
                dtstart = instance.component.propertyValue("DTSTART")
                if tzid is None and dtstart.getTimezoneID():
                    tzid = Timezone(tzid=dtstart.getTimezoneID())
                hasFloating |= dtstart.isDateOnly() or dtstart.floating()

                details.append(InstanceDetails(resid, uid, instance.start, instance.end, instance.component.getOrganizer(), instance.component.propertyValue("SUMMARY")))
                booked_instances += 1

            if self.options["verbose"] and not self.options["summary"]:
                if count == 1:
                    self.output.write("Instances".rjust(rjust) + "Current".rjust(rjust) + "Total".rjust(rjust) + "Complete".rjust(rjust) + "\n")
                if divmod(count, 100)[1] == 0:
                    self.output.write((
                        "\r" +
                        ("%s" % total_instances).rjust(rjust) +
                        ("%s" % count).rjust(rjust) +
                        ("%s" % total).rjust(rjust) +
                        ("%d%%" % safePercent(count, total)).rjust(rjust)
                    ).ljust(80))
                    self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(count, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"] and not self.options["summary"]:
            self.output.write((
                "\r" +
                ("%s" % total_instances).rjust(rjust) +
                ("%s" % count).rjust(rjust) +
                ("%s" % total).rjust(rjust) +
                ("%d%%" % safePercent(count, total)).rjust(rjust)
            ).ljust(80) + "\n")

        if not self.options["summary"]:
            self.logResult("Number of instances in time-range", total_instances)
            self.logResult("Number of booked instances", booked_instances)

        # Adjust floating and sort
        if hasFloating and tzid is not None:
            utc = Timezone.UTCTimezone
            for item in details:
                if item.start.floating():
                    item.start.setTimezone(tzid)
                    item.start.adjustTimezone(utc)
                if item.end.floating():
                    item.end.setTimezone(tzid)
                    item.end.adjustTimezone(utc)
        details.sort(key=lambda x: x.start)

        # Now look for double-bookings
        DoubleBookedDetails = collections.namedtuple("DoubleBookedDetails", ("resid1", "uid1", "resid2", "uid2", "start",))
        double_booked = []
        current = details[0] if details else None
        for next in details[1:]:
            if current.end > next.start and current.resid != next.resid and not (current.organizer == next.organizer and current.summary == next.summary):
                dt = next.start.duplicate()
                dt.adjustTimezone(self.tzid)
                double_booked.append(DoubleBookedDetails(current.resid, current.uid, next.resid, next.uid, dt,))
            current = next

        # Print table of results
        if double_booked and not self.options["summary"]:
            table = tables.Table()
            table.addHeader(("RID #1", "UID #1", "RID #2", "UID #2", "Start",))
            previous1 = None
            previous2 = None
            unique_events = 0
            for item in sorted(double_booked):
                if previous1 != item.resid1:
                    unique_events += 1
                resid1 = item.resid1 if previous1 != item.resid1 else "."
                uid1 = item.uid1 if previous1 != item.resid1 else "."
                resid2 = item.resid2 if previous2 != item.resid2 else "."
                uid2 = item.uid2 if previous2 != item.resid2 else "."
                table.addRow((
                    resid1,
                    uid1,
                    resid2,
                    uid2,
                    item.start,
                ))
                previous1 = item.resid1
                previous2 = item.resid2

            self.output.write("\n")
            self.logResult("Number of double-bookings", len(double_booked))
            self.logResult("Number of unique double-bookings", unique_events)
            table.printTable(os=self.output)

        self.results["Double-bookings"] = double_booked

        if self.options["verbose"] and not self.options["summary"]:
            diff_time = time.time() - t
            self.output.write("Time: %.2f s  Average: %.1f ms/resource\n" % (
                diff_time,
                safePercent(diff_time, total, 1000.0),
            ))

        returnValue(len(double_booked) != 0)


class DarkPurgeService(CalVerifyService):
    """
    Service which detects room/resource events that have an invalid organizer.
    """

    def title(self):
        return "Dark Purge Service"

    @inlineCallbacks
    def doAction(self):

        if not self.options["no-organizer"] and not self.options["invalid-organizer"] and not self.options["disabled-organizer"]:
            self.options["invalid-organizer"] = self.options["disabled-organizer"] = True

        self.output.write("\n---- Scanning calendar data ----\n")

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")
        self.now = DateTime.getNowUTC()
        self.start = self.options["start"] if "start" in self.options else DateTime.getToday()
        self.start.setDateOnly(False)
        self.start.setTimezone(self.tzid)
        self.fix = self.options["fix"]

        if self.options["verbose"] and self.options["summary"]:
            ot = time.time()

        # Check loop over uuid
        UUIDDetails = collections.namedtuple("UUIDDetails", ("uuid", "rname", "purged",))
        self.uuid_details = []
        if len(self.options["uuid"]) != 36:
            self.txn = self.store.newTransaction()
            if self.options["uuid"]:
                homes = yield self.getMatchingHomeUIDs(self.options["uuid"])
            else:
                homes = yield self.getAllHomeUIDs()
            yield self.txn.commit()
            self.txn = None
            uuids = []
            if self.options["verbose"]:
                self.output.write("%d uuids to check\n" % (len(homes,)))
            for uuid in sorted(homes):
                record = yield self.directoryService().recordWithUID(uuid)
                if record is not None and record.recordType in (CalRecordType.location, CalRecordType.resource):
                    uuids.append(uuid)
        else:
            uuids = [self.options["uuid"], ]
        if self.options["verbose"]:
            self.output.write("%d uuids to scan\n" % (len(uuids,)))

        count = 0
        for uuid in uuids:
            self.results = {}
            self.summary = []
            self.total = 0
            count += 1

            record = yield self.directoryService().recordWithUID(uuid)
            if record is None:
                continue
            if not record.thisServer() or not record.hasCalendars:
                continue

            rname = record.displayName

            if len(uuids) > 1 and not self.options["summary"]:
                self.output.write("\n\n-----------------------------\n")

            self.txn = self.store.newTransaction()

            if self.options["verbose"]:
                t = time.time()
            rows = yield self.getAllResourceInfoTimeRangeWithUUID(self.start, uuid)
            descriptor = "getAllResourceInfoTimeRangeWithUUID"

            yield self.txn.commit()
            self.txn = None

            if self.options["verbose"]:
                if not self.options["summary"]:
                    self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))
                else:
                    self.output.write("%s (%d/%d)" % (uuid, count, len(uuids),))
                    self.output.flush()

            self.total = len(rows)
            if not self.options["summary"]:
                self.logResult("UUID to process", uuid)
                self.logResult("Record name", rname)
                self.addSummaryBreak()
                self.logResult("Number of events to process", self.total)

            if rows:
                if not self.options["summary"]:
                    self.addSummaryBreak()
                purged = yield self.darkPurge(rows, uuid)
            else:
                purged = False

            self.uuid_details.append(UUIDDetails(uuid, rname, purged))

            if not self.options["summary"]:
                self.printSummary()
            else:
                self.output.write(" - %s\n" % ("Dark Events" if purged else "OK",))
                self.output.flush()

        if count == 0:
            self.output.write("Nothing to scan\n")

        if self.options["summary"]:
            table = tables.Table()
            table.addHeader(("GUID", "Name", "RID", "UID", "Organizer",))
            purged = 0
            for item in sorted(self.uuid_details):
                if not item.purged:
                    continue
                uuid = item.uuid
                rname = item.rname
                for detail in item.purged:
                    table.addRow((
                        uuid,
                        rname,
                        detail.resid,
                        detail.uid,
                        detail.organizer,
                    ))
                    uuid = ""
                    rname = ""
                    purged += 1
            table.addFooter(("Total", "%d" % (purged,), "", "", "",))
            self.output.write("\n")
            table.printTable(os=self.output)

            if self.options["verbose"]:
                self.output.write("%s time: %.1fs\n" % ("Summary", time.time() - ot,))

    @inlineCallbacks
    def darkPurge(self, rows, uuid):
        """
        Check each calendar resource by looking at any ORGANIER property value and verifying it is valid.
        """

        if not self.options["summary"]:
            self.output.write("\n---- Checking for dark events ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()

        Details = collections.namedtuple("Details", ("resid", "uid", "organizer",))

        count = 0
        total = len(rows)
        details = []
        fixed = 0
        rjust = 10
        for resid in rows:
            resid = resid[1]
            caldata = yield self.getCalendar(resid, self.fix)
            if caldata is None:
                if self.parseError:
                    returnValue((False, self.parseError))
                else:
                    returnValue((True, "Nothing to scan"))

            cal = Component(None, pycalendar=caldata)
            uid = cal.resourceUID()

            fail = False
            organizer = cal.getOrganizer()
            if organizer is None:
                if self.options["no-organizer"]:
                    fail = True
            else:
                principal = yield self.directoryService().recordWithCalendarUserAddress(organizer)
                # FIXME: Why the mix of records and principals here?
                if principal is None and organizer.startswith("urn:x-uid:"):
                    principal = yield self.directoryService().principalCollection.principalForUID(organizer[10:])
                if principal is None:
                    if self.options["invalid-organizer"]:
                        fail = True
                elif not principal.calendarsEnabled():
                    if self.options["disabled-organizer"]:
                        fail = True

            if fail:
                details.append(Details(resid, uid, organizer,))
                if self.fix:
                    yield self.removeEvent(resid)
                    fixed += 1

            if self.options["verbose"] and not self.options["summary"]:
                if count == 1:
                    self.output.write("Current".rjust(rjust) + "Total".rjust(rjust) + "Complete".rjust(rjust) + "\n")
                if divmod(count, 100)[1] == 0:
                    self.output.write((
                        "\r" +
                        ("%s" % count).rjust(rjust) +
                        ("%s" % total).rjust(rjust) +
                        ("%d%%" % safePercent(count, total)).rjust(rjust)
                    ).ljust(80))
                    self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(count, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"] and not self.options["summary"]:
            self.output.write((
                "\r" +
                ("%s" % count).rjust(rjust) +
                ("%s" % total).rjust(rjust) +
                ("%d%%" % safePercent(count, total)).rjust(rjust)
            ).ljust(80) + "\n")

        # Print table of results
        if not self.options["summary"]:
            self.logResult("Number of dark events", len(details))

        self.results["Dark Events"] = details
        if self.fix:
            self.results["Fix dark events"] = fixed

        if self.options["verbose"] and not self.options["summary"]:
            diff_time = time.time() - t
            self.output.write("Time: %.2f s  Average: %.1f ms/resource\n" % (
                diff_time,
                safePercent(diff_time, total, 1000.0),
            ))

        returnValue(details)


class MissingLocationService(CalVerifyService):
    """
    Service which detects room/resource events that have an ATTENDEE;CUTYPE=ROOM property but no Location.
    """

    def title(self):
        return "Missing Location Service"

    @inlineCallbacks
    def doAction(self):

        self.output.write("\n---- Scanning calendar data ----\n")

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")
        self.now = DateTime.getNowUTC()
        self.start = self.options["start"] if "start" in self.options else DateTime.getToday()
        self.start.setDateOnly(False)
        self.start.setTimezone(self.tzid)
        self.fix = self.options["fix"]

        if self.options["verbose"] and self.options["summary"]:
            ot = time.time()

        # Check loop over uuid
        UUIDDetails = collections.namedtuple("UUIDDetails", ("uuid", "rname", "missing",))
        self.uuid_details = []
        if len(self.options["uuid"]) != 36:
            self.txn = self.store.newTransaction()
            if self.options["uuid"]:
                homes = yield self.getMatchingHomeUIDs(self.options["uuid"])
            else:
                homes = yield self.getAllHomeUIDs()
            yield self.txn.commit()
            self.txn = None
            uuids = []
            if self.options["verbose"]:
                self.output.write("%d uuids to check\n" % (len(homes,)))
            for uuid in sorted(homes):
                record = yield self.directoryService().recordWithUID(uuid)
                if record is not None and record.recordType == CalRecordType.location:
                    uuids.append(uuid)
        else:
            uuids = [self.options["uuid"], ]
        if self.options["verbose"]:
            self.output.write("%d uuids to scan\n" % (len(uuids,)))

        count = 0
        for uuid in uuids:
            self.results = {}
            self.summary = []
            self.total = 0
            count += 1

            record = yield self.directoryService().recordWithUID(uuid)
            if record is None:
                continue
            if not record.thisServer() or not record.hasCalendars:
                continue

            rname = record.displayName

            if len(uuids) > 1 and not self.options["summary"]:
                self.output.write("\n\n-----------------------------\n")

            self.txn = self.store.newTransaction()

            if self.options["verbose"]:
                t = time.time()
            rows = yield self.getAllResourceInfoTimeRangeWithUUID(self.start, uuid)
            descriptor = "getAllResourceInfoTimeRangeWithUUID"

            yield self.txn.commit()
            self.txn = None

            if self.options["verbose"]:
                if not self.options["summary"]:
                    self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))
                else:
                    self.output.write("%s (%d/%d)" % (uuid, count, len(uuids),))
                    self.output.flush()

            self.total = len(rows)
            if not self.options["summary"]:
                self.logResult("UUID to process", uuid)
                self.logResult("Record name", rname)
                self.addSummaryBreak()
                self.logResult("Number of events to process", self.total)

            if rows:
                if not self.options["summary"]:
                    self.addSummaryBreak()
                missing = yield self.missingLocation(rows, uuid, rname)
            else:
                missing = False

            self.uuid_details.append(UUIDDetails(uuid, rname, missing))

            if not self.options["summary"]:
                self.printSummary()
            else:
                self.output.write(" - %s\n" % ("Bad Events" if missing else "OK",))
                self.output.flush()

        if count == 0:
            self.output.write("Nothing to scan\n")

        if self.options["summary"]:
            table = tables.Table()
            table.addHeader(("GUID", "Name", "RID", "UID",))
            missing = 0
            for item in sorted(self.uuid_details):
                if not item.missing:
                    continue
                uuid = item.uuid
                rname = item.rname
                for detail in item.missing:
                    table.addRow((
                        uuid,
                        rname,
                        detail.resid,
                        detail.uid,
                    ))
                    uuid = ""
                    rname = ""
                    missing += 1
            table.addFooter(("Total", "%d" % (missing,), "", "", "",))
            self.output.write("\n")
            table.printTable(os=self.output)

            if self.options["verbose"]:
                self.output.write("%s time: %.1fs\n" % ("Summary", time.time() - ot,))

    @inlineCallbacks
    def missingLocation(self, rows, uuid, rname):
        """
        Check each calendar resource by looking at any ORGANIZER property value and verifying it is valid.
        """

        if not self.options["summary"]:
            self.output.write("\n---- Checking for missing location events ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()

        Details = collections.namedtuple("Details", ("resid", "uid",))

        count = 0
        total = len(rows)
        details = []
        fixed = 0
        rjust = 10
        cuaddr = "urn:x-uid:{}".format(uuid)
        for resid in rows:
            resid = resid[1]
            caldata = yield self.getCalendar(resid, self.fix)
            if caldata is None:
                if self.parseError:
                    returnValue((False, self.parseError))
                else:
                    returnValue((True, "Nothing to scan"))

            cal = Component(None, pycalendar=caldata)
            uid = cal.resourceUID()

            fail = False
            if cal.getOrganizer() is not None:
                for comp in cal.subcomponents():
                    if comp.name() != "VEVENT":
                        continue

                    # Look for matching location property
                    location = comp.propertyValue("LOCATION")
                    if location is not None:
                        items = location.split(";")
                        has_location = rname in items
                    else:
                        has_location = False

                    # Look for matching attendee property
                    has_attendee = cuaddr in comp.getAttendees()

                    if not has_location and has_attendee:
                        fail = True
                        break

            if fail:
                details.append(Details(resid, uid,))
                if self.fix:
                    yield self.fixCalendarData(cal, rname, cuaddr, resid)
                    fixed += 1

            if self.options["verbose"] and not self.options["summary"]:
                if count == 1:
                    self.output.write("Current".rjust(rjust) + "Total".rjust(rjust) + "Complete".rjust(rjust) + "\n")
                if divmod(count, 100)[1] == 0:
                    self.output.write((
                        "\r" +
                        ("%s" % count).rjust(rjust) +
                        ("%s" % total).rjust(rjust) +
                        ("%d%%" % safePercent(count, total)).rjust(rjust)
                    ).ljust(80))
                    self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(count, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"] and not self.options["summary"]:
            self.output.write((
                "\r" +
                ("%s" % count).rjust(rjust) +
                ("%s" % total).rjust(rjust) +
                ("%d%%" % safePercent(count, total)).rjust(rjust)
            ).ljust(80) + "\n")

        # Print table of results
        if not self.options["summary"]:
            self.logResult("Number of bad events", len(details))

        self.results["Bad Events"] = details
        if self.fix:
            self.results["Fix bad events"] = fixed

        if self.options["verbose"] and not self.options["summary"]:
            diff_time = time.time() - t
            self.output.write("Time: %.2f s  Average: %.1f ms/resource\n" % (
                diff_time,
                safePercent(diff_time, total, 1000.0),
            ))

        returnValue(details)

    @inlineCallbacks
    def fixCalendarData(self, cal, rname, cuaddr, location_resid):
        """
        Fix problems in calendar data using store APIs.
        """

        result = True
        message = ""

        # Extract organizer (strip off urn:x-uid:) and UID
        organizer = cal.getOrganizer()[10:]
        uid = cal.resourceUID()
        _ignore_calendar, resid, _ignore_created, _ignore_modified = yield self.getCalendarForOwnerByUID(organizer, uid)
        if resid is None:
            # No organizer copy exists - it is safe to delete the location resource
            yield self.removeEvent(location_resid)
            returnValue((result, message,))

        # Get the organizer's calendar object and data
        homeID, calendarID = yield self.getAllResourceInfoForResourceID(resid)
        home = yield self.txn.calendarHomeWithResourceID(homeID)
        calendar = yield home.childWithID(calendarID)
        calendarObj = yield calendar.objectResourceWithID(resid)

        try:
            component = yield calendarObj.componentForUser()
        except InternalDataStoreError:
            returnValue((False, "Failed parse: "))

        # Fix each component (need to dup component when modifying)
        component = component.duplicate()
        for comp in component.subcomponents():
            if comp.name() != "VEVENT":
                continue
            location = comp.propertyValue("LOCATION")
            if location is None:
                # Just add the location name back in
                comp.addProperty(Property("LOCATION", rname))
            else:
                # Remove the matching ATTENDEE property
                attendee = comp.getAttendeeProperty((cuaddr,))
                if attendee is not None:
                    comp.removeProperty(attendee)

        # Write out fix, commit and get a new transaction
        try:
            yield calendarObj.setComponent(component)
        except Exception, e:
            print(e, component)
            print(traceback.print_exc())
            result = False
            message = "Exception fix: "
        yield self.txn.commit()
        self.txn = self.store.newTransaction()

        returnValue((result, message,))


class EventSplitService(CalVerifyService):
    """
    Service which splits a recurring event at a specific date-time value.
    """

    def title(self):
        return "Event Split Service"

    @inlineCallbacks
    def doAction(self):
        """
        Split a resource using either its path or resource id.
        """

        self.txn = self.store.newTransaction()

        path = self.options["path"]
        if path.startswith("/calendars/__uids__/"):
            try:
                pathbits = path.split("/")
            except TypeError:
                printusage("Not a valid calendar object resource path: %s" % (path,))
            if len(pathbits) != 6:
                printusage("Not a valid calendar object resource path: %s" % (path,))
            homeName = pathbits[3]
            calendarName = pathbits[4]
            resourceName = pathbits[5]

            resid = yield self.getResourceID(homeName, calendarName, resourceName)
            if resid is None:
                yield self.txn.commit()
                self.txn = None
                self.output.write("\n")
                self.output.write("Path does not exist. Nothing split.\n")
                returnValue(None)
            resid = int(resid)
        else:
            try:
                resid = int(path)
            except ValueError:
                printusage("path argument must be a calendar object path or an SQL resource-id")

        calendarObj = yield CalendarStoreFeatures(self.txn._store).calendarObjectWithID(self.txn, resid)
        ical = yield calendarObj.component()

        # Must be the ORGANIZER's copy
        organizer = ical.getOrganizer()
        if organizer is None:
            printusage("Calendar object has no ORGANIZER property - cannot split")

        # Only allow organizers to split
        scheduler = ImplicitScheduler()
        is_attendee = (yield scheduler.testAttendeeEvent(calendarObj.calendar(), calendarObj, ical,))
        if is_attendee:
            printusage("Calendar object is not owned by the ORGANIZER - cannot split")

        if self.options["summary"]:
            result = self.doSummary(ical)
        else:
            result = yield self.doSplit(resid, calendarObj, ical)

        returnValue(result)

    def doSummary(self, ical):
        """
        Print a summary of the recurrence instances of the specified event.

        @param ical: calendar to process
        @type ical: L{Component}
        """
        self.output.write("\n---- Calendar resource instances ----\n")

        # Find the instance RECURRENCE-ID where a split is going to happen
        now = DateTime.getNowUTC()
        now.offsetDay(1)
        instances = ical.cacheExpandedTimeRanges(now)
        instances = sorted(instances.instances.values(), key=lambda x: x.start)
        for instance in instances:
            self.output.write(instance.rid.getText() + (" *\n" if instance.overridden else "\n"))

    @inlineCallbacks
    def doSplit(self, resid, calendarObj, ical):
        rid = self.options["rid"]
        try:
            if rid[-1] != "Z":
                raise ValueError
            rid = DateTime.parseText(rid)
        except ValueError:
            printusage("rid must be a valid UTC date-time value: 'YYYYMMDDTHHMMSSZ'")

        self.output.write("\n---- Splitting calendar resource ----\n")

        # Find actual RECURRENCE-ID of split
        splitter = iCalSplitter(1024, 14)
        rid = splitter.whereSplit(ical, break_point=rid, allow_past_the_end=False)
        if rid is None:
            printusage("rid is not a valid recurrence instance")

        self.output.write("\n")
        self.output.write("Actual RECURRENCE-ID: %s.\n" % (rid,))

        oldObj = yield calendarObj.split(rid=rid)
        oldUID = oldObj.uid()
        oldRelated = (yield oldObj.component()).mainComponent().propertyValue("RELATED-TO")

        self.output.write("\n")
        self.output.write("Split Resource: %s at %s, old UID: %s.\n" % (resid, rid, oldUID,))

        yield self.txn.commit()
        self.txn = None

        returnValue((oldUID, oldRelated,))


class UpgradeDataService(CalVerifyService):
    """
    Service which scans calendar data.
    """

    USE_FAST_MODE = True
    OLD_CUADDR_PREFIX = "urn:uuid:"
    NEW_CUADDR_PREFIX = "urn:x-uid:"

    def title(self):
        return "Upgrade Data Service"

    @inlineCallbacks
    def doAction(self):

        self.output.write("\n---- Scanning calendar data ----\n")

        self.now = DateTime.getNowUTC()
        self.start = DateTime.getToday()
        self.start.setDateOnly(False)
        self.end = self.start.duplicate()
        self.end.offsetYear(1)
        self.fix = self.options["fix"]

        self.tzid = Timezone(tzid=self.options["tzid"] if self.options["tzid"] else "America/Los_Angeles")

        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()
        descriptor = None
        if self.options["uuid"]:
            rows = yield self.getAllResourceInfoWithUUID(self.options["uuid"], inbox=True)
            descriptor = "getAllResourceInfoWithUUID"
        elif self.options["uid"]:
            rows = yield self.getAllResourceInfoWithUID(self.options["uid"], inbox=True)
            descriptor = "getAllResourceInfoWithUID"
        else:
            rows = yield self.getAllResourceInfo(inbox=True)
            descriptor = "getAllResourceInfo"

        yield self.txn.commit()
        self.txn = None

        if self.options["verbose"]:
            self.output.write("%s time: %.1fs\n" % (descriptor, time.time() - t,))

        self.total = len(rows)
        self.logResult("Number of events to process", self.total)
        self.addSummaryBreak()

        yield self.calendarDataUpgrade(rows)

        self.printSummary()

    @inlineCallbacks
    def calendarDataUpgrade(self, rows):
        """
        Check each calendar resource for valid iCalendar data.
        """

        self.output.write("\n---- Upgrading each calendar object resource ----\n")
        self.txn = self.store.newTransaction()

        if self.options["verbose"]:
            t = time.time()

        results_bad = []
        count = 0
        total = len(rows)
        upgradelen = 0
        badlen = 0
        rjust = 10
        for owner, resid, uid, _ignore_calname, _ignore_md5, _ignore_organizer, _ignore_created, _ignore_modified in rows:
            try:
                calendarObj = yield CalendarStoreFeatures(self.txn._store).calendarObjectWithID(self.txn, resid)
                if calendarObj._dataversion < calendarObj._currentDataVersion:
                    upgradelen += 1

                    if UpgradeDataService.USE_FAST_MODE:
                        # Read data from calendar object and manually upgrade the calendar user
                        # addresses only
                        text = yield calendarObj._text()
                        component = Component.fromString(text)

                        for subcomponent in component.subcomponents(ignore=True):
                            for prop in itertools.chain(
                                subcomponent.properties("ORGANIZER"),
                                subcomponent.properties("ATTENDEE"),
                            ):
                                cuaddr = prop.value()
                                if cuaddr.startswith(UpgradeDataService.OLD_CUADDR_PREFIX):
                                    prop.setValue(cuaddr.replace(UpgradeDataService.OLD_CUADDR_PREFIX, UpgradeDataService.NEW_CUADDR_PREFIX))

                        # Do the update right now (also suppress instance indexing)
                        calendarObj._dataversion = calendarObj._currentDataVersion
                        component.noInstanceIndexing = True
                        yield calendarObj.updateDatabase(component)
                    else:
                        # Read component and force and update if needed
                        yield calendarObj.component(doUpdate=True)

                result = True
            except Exception, e:
                result = False
                message = "Exception when reading resource"
                if self.options["verbose"]:
                    print(e)
            if not result:
                results_bad.append((owner, uid, resid, message))
                badlen += 1
            count += 1
            if self.options["verbose"]:
                if count == 1:
                    self.output.write("Upgraded".rjust(rjust) + "Bad".rjust(rjust) + "Current".rjust(rjust) + "Total".rjust(rjust) + "Complete".rjust(rjust) + "\n")
                if divmod(count, 100)[1] == 0:
                    self.output.write((
                        "\r" +
                        ("%s" % upgradelen).rjust(rjust) +
                        ("%s" % badlen).rjust(rjust) +
                        ("%s" % count).rjust(rjust) +
                        ("%s" % total).rjust(rjust) +
                        ("%d%%" % safePercent(count, total)).rjust(rjust)
                    ).ljust(80))
                    self.output.flush()

            # To avoid holding locks on all the rows scanned, commit every 100 resources
            if divmod(count, 100)[1] == 0:
                yield self.txn.commit()
                self.txn = self.store.newTransaction()

        yield self.txn.commit()
        self.txn = None
        if self.options["verbose"]:
            self.output.write((
                "\r" +
                ("%s" % upgradelen).rjust(rjust) +
                ("%s" % badlen).rjust(rjust) +
                ("%s" % count).rjust(rjust) +
                ("%s" % total).rjust(rjust) +
                ("%d%%" % safePercent(count, total)).rjust(rjust)
            ).ljust(80) + "\n")

        # Print table of results
        table = tables.Table()
        table.addHeader(("Owner", "Event UID", "RID", "Problem",))
        for item in sorted(results_bad, key=lambda x: (x[0], x[1])):
            owner, uid, resid, message = item
            owner_record = yield self.directoryService().recordWithUID(owner)
            table.addRow((
                "%s/%s (%s)" % (owner_record.recordType if owner_record else "-", owner_record.shortNames[0] if owner_record else "-", owner,),
                uid,
                resid,
                message,
            ))

        self.output.write("\n")
        self.logResult("Upgraded iCalendar data", upgradelen, total)
        self.logResult("Bad iCalendar data", len(results_bad), total)
        self.results["Upgraded iCalendar data"] = upgradelen
        self.results["Bad iCalendar data"] = results_bad
        table.printTable(os=self.output)

        if self.options["verbose"]:
            diff_time = time.time() - t
            self.output.write("Time: %.2f s  Average: %.1f ms/resource\n" % (
                diff_time,
                safePercent(diff_time, total, 1000.0),
            ))


def main(argv=sys.argv, stderr=sys.stderr, reactor=None):

    if reactor is None:
        from twisted.internet import reactor
    options = CalVerifyOptions()
    try:
        options.parseOptions(argv[1:])
    except usage.UsageError, e:
        printusage(e)

    try:
        output = options.openOutput()
    except IOError, e:
        stderr.write("Unable to open output file for writing: %s\n" % (e))
        sys.exit(1)

    def makeService(store):
        from twistedcaldav.config import config
        config.TransactionTimeoutSeconds = 0
        if options["nuke"]:
            return NukeService(store, options, output, reactor, config)
        elif options["missing"]:
            return OrphansService(store, options, output, reactor, config)
        elif options["ical"]:
            return BadDataService(store, options, output, reactor, config)
        elif options["mismatch"]:
            return SchedulingMismatchService(store, options, output, reactor, config)
        elif options["double"]:
            return DoubleBookingService(store, options, output, reactor, config)
        elif options["dark-purge"]:
            return DarkPurgeService(store, options, output, reactor, config)
        elif options["missing-location"]:
            return MissingLocationService(store, options, output, reactor, config)
        elif options["split"]:
            return EventSplitService(store, options, output, reactor, config)
        elif options["upgrade"]:
            return UpgradeDataService(store, options, output, reactor, config)
        else:
            printusage("Invalid operation")
            sys.exit(1)

    utilityMain(options['config'], makeService, reactor)

if __name__ == '__main__':
    main()