How to open a lightning:datatable in a modal/popup.

Try this one.I have tried this in my org,am able to open datatable in popup.

cmp
<aura:component controller = "AccountController">
    <aura:attribute name="isOpen" type="boolean" default="false"/>
    <aura:attribute type="Account[]" name="acctList"/>
    <aura:attribute name="mycolumns" type="List"/>
    <aura:handler name="init" value="{!this}" action="{!c.fetchAcc}"/>
    <div class="slds-m-around_xx-large">
        <lightning:input type="checkbox" label="Required option" name="input2" checked="false" onchange = "{!c.openModel}"/>
        <aura:if isTrue="{!v.isOpen}">
            <section role="dialog" tabindex="-1" aria-labelledby="modal-heading-01" aria-modal="true" aria-describedby="modal-content-id-1" class="slds-modal slds-fade-in-open">
                <div class="slds-modal__container">
                    <header class="slds-modal__header">
                        <lightning:buttonIcon iconName="utility:close"
                                              onclick="{! c.closeModel }"
                                              alternativeText="close"
                                              variant="bare-inverse"
                                              class="slds-modal__close"/>
                        <h2 id="modal-heading-01" class="slds-text-heading_medium slds-hyphenate">About Sfdcmonkey.com</h2>
                    </header>
                    <div style="height: 300px">
                        <lightning:datatable data="{! v.acctList }"
                                             columns="{! v.mycolumns }"
                                             keyField="id"
                                             hideCheckboxColumn="true"/>
                    </div>
                    <footer class="slds-modal__footer">
                        <lightning:button variant="neutral" 
                                          label="Cancel"
                                          title="Cancel"
                                          onclick="{! c.closeModel }"/>
                    </footer>
                </div>
            </section>
            <div class="slds-backdrop slds-backdrop_open"></div>
        </aura:if>
    </div>
</aura:component>
controller:
({
    fetchAcc : function(component, event, helper) {
        helper.fetchAccHelper(component, event, helper);
    },
    openModel: function(component, event, helper) {
        component.set("v.isOpen", true);
    },
    closeModel: function(component, event, helper) { 
        component.set("v.isOpen", false);
    },
})
helper:
({
    fetchAccHelper : function(component, event, helper) {
        component.set('v.mycolumns', [
            {label: 'Account Name', fieldName: 'Name', type: 'text'},
            {label: 'Industry', fieldName: 'Industry', type: 'text'},
            {label: 'Phone', fieldName: 'Phone', type: 'Phone'},
            {label: 'Website', fieldName: 'Website', type: 'url '}
        ]);
        var action = component.get("c.fetchAccounts");
        action.setParams({
        });
        action.setCallback(this, function(response){
            var state = response.getState();
            if (state === "SUCCESS") {
                component.set("v.acctList", response.getReturnValue());
            }
        });
        $A.enqueueAction(action);
    }
})
public class AccountController {
@AuraEnabled
    public static List <Account> fetchAccounts() {
        //Qyery 10 accounts
        List<Account> accList = [SELECT Id, Name, BillingState, 
                                    Website, Phone, Industry, Type from Account LIMIT 10];
        //return list of accounts
        return accList;
    }
}
application:
<aura:application extends="force:slds">
    <c:ModalPopup/>
</aura:application>

Please refer below link which might help you